/** * Creates a course if it does not exist * @param $name Name of the course * @param Initiative $initiative * @param Institution $ins * @return Course */ public function createCourseIfNotExists($name, Initiative $initiative, Institution $ins = null, Stream $stream) { // Check if course exists $em = $this->scraper->getManager(); $courseRepository = $em->getRepository('ClassCentralSiteBundle:Course'); $course = $courseRepository->findOneBy(array('name' => $name, 'initiative' => $initiative->getId())); // Course exists if ($course) { return $course; } $course = new Course(); $course->setName($name); $course->setInitiative($initiative); if ($ins) { $course->addInstitution($ins); } $course->setStream($stream); // Check if course is to be created if ($this->scraper->doModify() && $this->scraper->doCreate()) { $em->persist($course); $em->flush(); $this->scraper->out("COURSE {$name} created for initiative " . $initiative->getName()); } return $course; }
/** * Build a doctrine Course Entity out of a csv row * @param $row * @return Course */ public function getCourseEntity($row) { $course = new Course(); $course->setName($row[0]); $course->setDescription($row[1]); $course->setVideoIntro(str_replace('http', 'https', $row[4])); $course->setUrl($row[5]); $course->setShortName($this->getCourseId($row[5])); $course->setInitiative($this->initiative); // Set the language to arabic $langMap = $this->dbHelper->getLanguageMap(); $course->setLanguage($langMap['Arabic']); // Set the default stream as humanities $defaultStream = $this->dbHelper->getStreamBySlug('humanities'); $course->setStream($defaultStream); // Calculate the length of the course $start = new \DateTime($row[2]); $end = new \DateTime($row[3]); $length = ceil($start->diff($end)->days / 7); $course->setLength($length); return $course; }
public function migrate() { $this->output->writeln("Getting Started with migration version 1"); // Get all offerings $em = $this->container->get('Doctrine')->getManager(); $offerings = $em->getRepository('ClassCentralSiteBundle:Offering')->findAll(); foreach ($offerings as $offering) { $name = $offering->getName(); $initiative = null; if ($offering->getInitiative()) { $initiative = $offering->getInitiative(); } $stream = $offering->getCourse()->getStream(); // Check if the course name and initiative exist $initiative_id = $initiative ? $initiative->getID() : null; $course = $em->getRepository('ClassCentralSiteBundle:Course')->findOneBy(array('name' => $name, 'initiative' => $initiative_id)); if (!$course) { // Course does not exist. Create the course //$this->output->writeln("NOT FOUND"); $course = new Course(); $course->setName($name); $course->setInitiative($initiative); $course->setStream($stream); $em->persist($course); $em->flush(); $this->output->writeln("Course '" . $course->getName() . "' created with " . $course->getId()); } else { $this->output->writeln("Course '" . $course->getName() . " already exists with " . $course->getId()); } // Update the course id in offering $offering->setCourse($course); $em->persist($offering); $em->flush(); } // Delete all the courses with course id 100 $em->createQuery("DELETE FROM ClassCentralSiteBundle:Course c WHERE c.initiative=100")->execute(); // Delete initative $em->createQuery("DELETE FROM ClassCentralSiteBundle:Initiative i WHERE i.id = 100")->execute(); }
/** * Given an array built from edX csv returns a course entity * @param array $c */ private function getCourseEntity($c = array()) { $defaultStream = $this->dbHelper->getStreamBySlug('cs'); $langMap = $this->dbHelper->getLanguageMap(); $defaultLanguage = $langMap['English']; $course = new Course(); $course->setShortName($this->getShortName($c)); $course->setInitiative($this->initiative); $course->setName($c['course-code'] . ': ' . $c['title']); $course->setDescription($c['description']); $course->setLongDescription(nl2br($c['description'])); $course->setLanguage($defaultLanguage); $course->setStream($defaultStream); // Default to Computer Science $course->setVideoIntro($c['course-video-youtube']); $course->setUrl($c['link']); $course->setCertificate(false); $course->setVerifiedCertificate($c['course-verified']); // Calculate length $length = null; if (!empty($c['course-end'])) { $start = new \DateTime($c['course-start']); $end = new \DateTime($c['course-end']); $length = ceil($start->diff($end)->days / 7); } $course->setLength($length); return $course; }
private function getOnDemandCourse($data = array()) { $dbLanguageMap = $this->dbHelper->getLanguageMap(); $course = new Course(); $course->setShortName(substr('coursera_' . $data['elements'][0]['slug'], 0, 49)); $course->setInitiative($this->initiative); $course->setName($data['elements'][0]['name']); $course->setDescription($data['elements'][0]['description']); $course->setLongDescription(nl2br($data['elements'][0]['description'])); $course->setStream($this->dbHelper->getStreamBySlug('cs')); // Default to Computer Science $course->setUrl('https://www.coursera.org/learn/' . $data['elements'][0]['slug']); $lang = self::$languageMap[$data['elements']['0']['primaryLanguageCodes'][0]]; if (isset($dbLanguageMap[$lang])) { $course->setLanguage($dbLanguageMap[$lang]); } else { $this->out("Language not found " . $data['elements']['0']['primaryLanguageCodes'][0]); } $course->setCertificate(false); $course->setVerifiedCertificate($data['elements'][0]['isVerificationEnabled']); // Add the university foreach ($data['linked']['partners.v1'] as $university) { $ins = new Institution(); $ins->setName($university['name']); $ins->setIsUniversity(true); $ins->setSlug($university['shortName']); $course->addInstitution($this->dbHelper->createInstitutionIfNotExists($ins)); } foreach ($data['linked']['instructors.v1'] as $courseraInstructor) { if (!empty($courseraInstructor['fullName'])) { $insName = $courseraInstructor['fullName']; } else { $insName = $courseraInstructor['firstName'] . ' ' . $courseraInstructor['lastName']; } $course->addInstructor($this->dbHelper->createInstructorIfNotExists($insName)); } // Get Course Details like Syllabus and length $courseDetails = json_decode(file_get_contents(sprintf(self::ONDEMAND_OPENCOURSE_API, $data['elements'][0]['slug'])), true); if (!empty($courseDetails)) { $syllabus = ''; foreach ($courseDetails['courseMaterial']['elements'] as $item) { $syllabus .= "<b>{$item['name']}</b><br/>{$item['description']}<br/><br/>"; } $course->setSyllabus($syllabus); } // Calculate the length of the course $schedule = json_decode(file_get_contents(sprintf(self::ONDEMAND_COURSE_SCHEDULE, $data['elements'][0]['id'])), true); if (!empty($schedule)) { $length = 0; foreach ($schedule['elements'][0]['defaultSchedule']['periods'] as $period) { $length += $period['numberOfWeeks']; } if ($length > 0) { $course->setLength($length); } } return $course; }
private function getCourseEntity($udacityCourse = array()) { $defaultStream = $this->dbHelper->getStreamBySlug('cs'); $langMap = $this->dbHelper->getLanguageMap(); $defaultLanguage = $langMap['English']; $course = new Course(); $course->setShortName(substr('udacity_' . $udacityCourse['slug'], 0, 50)); $course->setInitiative($this->initiative); $course->setName($udacityCourse['title']); $course->setDescription($udacityCourse['short_summary']); $course->setLanguage($defaultLanguage); $course->setStream($defaultStream); // Default to Computer Science $course->setCertificate(false); $course->setUrl($udacityCourse['homepage']); $course->setSyllabus(nl2br($udacityCourse['syllabus'])); $course->setWorkloadMin(6); $course->setWorkloadMax(6); // Calculate length $length = null; $expectedDuration = $udacityCourse['expected_duration']; if ($udacityCourse['expected_duration_unit'] == 'months') { $length = $expectedDuration * 4; } elseif ($udacityCourse['expected_duration_unit'] == 'weeks') { $length = $expectedDuration; } $course->setLength($length); // Calculate Description $course->setLongDescription(nl2br($udacityCourse['summary'] . '<br/><br/><b>Why Take This Course?</b><br/>' . $udacityCourse['expected_learning'])); // Intro Video if (!empty($udacityCourse['teaser_video']['youtube_url'])) { $course->setVideoIntro($udacityCourse['teaser_video']['youtube_url']); } return $course; }
public function scrape() { $em = $this->getManager(); // Array of offerings created or updated $offerings = array(); $this->out("Scraping " . $this->initiative->getName()); // Step 1: Getting a list of course URLs $this->out("Getting a list of course pages"); $urls = $this->getListOfCoursePages(); $urlsCount = count($urls); // Step 2: Go through the page and create/update offering $this->out("Number of courses found: {$urlsCount}"); $this->out("Gathering details about each course"); $courseDetails = array(); foreach ($urls as $url) { if (!$url) { continue; } $courseDetail = array(); $this->domParser->load(file_get_contents(self::BASE_URL . $url)); // Ignore self paced if (!$this->domParser->find('h2.offering_dates_date', 0)) { continue; } // Get Name and shortName $nameString = $this->domParser->find('h1.page-title', 0)->plaintext; $openBracketPosition = strpos($nameString, '('); $closeBracketPosition = strpos($nameString, ')'); $courseDetail['name'] = substr($nameString, 0, $openBracketPosition - 1); $courseDetail['shortName'] = substr($nameString, $openBracketPosition + 1, $closeBracketPosition - $openBracketPosition - 1); if ($courseDetail['name'] == 'Introduction to Nursing in Healthcar') { $courseDetail['name'] = 'Introduction to Nursing in Healthcare'; $courseDetail['shortName'] = 'IntroNur'; } // Get the video id from the url // eg. www.youtube.com/embed/Bw8HkjGQb3U?wmode=opaque&rel=0&showinfo=0 $youtubeIdPosition = 31; $video = 'http://' . $this->domParser->find('iframe.media-youtube-player', 0)->src; $questionMarkPosition = strpos($video, '?'); $courseDetail['video'] = 'http://www.youtube.com/watch?v=' . substr($video, $youtubeIdPosition, $questionMarkPosition - $youtubeIdPosition); $instructors = trim($this->domParser->find('div[id=subject-teacher-tagline]', 0)->plaintext); // Remove the 'by' $instructors = substr($instructors, 3); $courseDetail['instructors'] = explode(' & ', $instructors); $courseDetail['desc'] = $this->domParser->find('div.offering_body', 0)->plaintext; $courseDetail['start_date'] = $this->domParser->find('h2.offering_dates_date', 0)->plaintext; $courseDetail['end_date'] = $this->domParser->find('h2.offering_dates_date', 1)->plaintext; $courseDetail['url'] = $url; print_r($courseDetail); $courseDetails[] = $courseDetail; $this->domParser->clear(); } $this->out(count($courseDetails) . ' course pages found'); // Default stream $stream = $this->dbHelper->getStreamBySlug('business'); $this->out("Default stream is " . $stream->getName()); foreach ($courseDetails as $courseDetail) { /** * Taking a shortcut here. Check if a course is created or not. If it isn't create the * course,offering, etc. Updates are ignored * TODO: Not take a shortcut */ // Build a course object $course = new Course(); $courseShortName = 'open2study_' . $courseDetail['shortName']; $course->setShortName($courseShortName); $course->setInitiative($this->initiative); $course->setName($courseDetail['name']); $course->setDescription($courseDetail['desc']); $course->setStream($stream); // Default to Business $course->setVideoIntro($courseDetail['video']); $course->setUrl(self::BASE_URL . $courseDetail['url']); $dbCourse = $this->dbHelper->getCourseByShortName($courseShortName); if (!$dbCourse) { if ($this->doCreate()) { // New course $this->out("NEW COURSE - " . $course->getName()); if ($this->doModify()) { foreach ($courseDetail['instructors'] as $instructor) { $course->addInstructor($this->dbHelper->createInstructorIfNotExists($instructor)); } $em->persist($course); $em->flush(); } } } else { $course = $dbCourse; } // Check if offering exists $shortName = $this->getOfferingShortName($courseDetail); $offering = $this->dbHelper->getOfferingByShortName($shortName); if ($offering) { continue; } // Check if create offering is oon if (!$this->doCreate()) { $offerings[] = $offering; // Add it to the offerings table continue; } $offering = new Offering(); $offering->setCourse($course); $offering->setStartDate(\DateTime::createFromFormat("d/m/Y", $courseDetail['start_date'])); $offering->setEndDate(\DateTime::createFromFormat("d/m/Y", $courseDetail['end_date'])); $offering->setStatus(Offering::START_DATES_KNOWN); $offering->setLength(4); $offering->setShortName($shortName); $offering->setUrl(self::BASE_URL . $courseDetail['url']); $offering->setVideoIntro($courseDetail['video']); $offering->setSearchDesc($courseDetail['desc']); $offering->setCreated(new \DateTime()); if ($this->doModify()) { try { $em->persist($offering); $em->flush(); $this->out("OFFERING {$courseDetail['name']} created"); } catch (\Exception $e) { $this->out("OFFERING {$courseDetail['name']} creation FAILED"); } } $offerings[] = $offering; } return $offerings; }
public function getCourse($canvasCourse) { $dbLanguageMap = $this->dbHelper->getLanguageMap(); $course = new Course(); $course->setName($canvasCourse['title']); $course->setInitiative($this->initiative); $course->setDescription($canvasCourse['teaser']); $course->setUrl($canvasCourse['url']); $course->setLanguage($dbLanguageMap['English']); $course->setStream($this->dbHelper->getStreamBySlug('cs')); // Default to Computer Science $course->setShortName('canvas_' . $this->getSlug($canvasCourse['path'])); return $course; }
private function getOnDemandCourse($data = array()) { $dbLanguageMap = $this->dbHelper->getLanguageMap(); $course = new Course(); $course->setShortName(substr('coursera_' . $data['elements'][0]['slug'], 0, 49)); $course->setInitiative($this->initiative); $course->setName($data['elements'][0]['name']); $course->setDescription($data['elements'][0]['description']); $course->setLongDescription(nl2br($data['elements'][0]['description'])); $course->setStream($this->dbHelper->getStreamBySlug('cs')); // Default to Computer Science $course->setUrl('https://www.coursera.org/learn/' . $data['elements'][0]['slug']); $lang = self::$languageMap[$data['elements']['0']['primaryLanguageCodes'][0]]; if (isset($dbLanguageMap[$lang])) { $course->setLanguage($dbLanguageMap[$lang]); } else { $this->out("Language not found " . $data['elements']['0']['primaryLanguageCodes'][0]); } $course->setCertificate(false); $course->setVerifiedCertificate($data['elements'][0]['isVerificationEnabled']); // Add the university foreach ($data['linked']['partners.v1'] as $university) { $ins = new Institution(); $ins->setName($university['name']); $ins->setIsUniversity(true); $ins->setSlug($university['shortName']); $course->addInstitution($this->dbHelper->createInstitutionIfNotExists($ins)); } foreach ($data['linked']['instructors.v1'] as $courseraInstructor) { if (!empty($courseraInstructor['fullName'])) { $insName = $courseraInstructor['fullName']; } else { $insName = $courseraInstructor['firstName'] . ' ' . $courseraInstructor['lastName']; } $course->addInstructor($this->dbHelper->createInstructorIfNotExists($insName)); } return $course; }
private function getCourseEntity($c = array()) { $defaultStream = $this->dbHelper->getStreamBySlug('cs'); $langMap = $this->dbHelper->getLanguageMap(); $defaultLanguage = $langMap['English']; $course = new Course(); $course->setShortName($c['uuid']); $course->setInitiative($this->initiative); $course->setName($c['name']); $course->setDescription($c['introduction']); $course->setLongDescription($c['description']); $course->setLanguage($defaultLanguage); $course->setStream($defaultStream); // Default to Computer Science $course->setUrl($c['url']); $course->setCertificate($c['has_certificates']); $course->setWorkloadMin($c['hours_per_week']); $course->setWorkloadMax($c['hours_per_week']); // Get the length if ($c['runs']) { $course->setLength($c['runs'][0]['duration_in_weeks']); } return $course; }