/** * Given an array built from edX csv returns a course entity * @param array $c */ private function getCourseEntity($c = array()) { $defaultStream = $this->dbHelper->getStreamBySlug('cs'); $langMap = $this->dbHelper->getLanguageMap(); $defaultLanguage = $langMap['English']; $course = new Course(); $course->setShortName($this->getShortName($c)); $course->setInitiative($this->initiative); $course->setName($c['course-code'] . ': ' . $c['title']); $course->setDescription($c['description']); $course->setLongDescription(nl2br($c['description'])); $course->setLanguage($defaultLanguage); $course->setStream($defaultStream); // Default to Computer Science $course->setVideoIntro($c['course-video-youtube']); $course->setUrl($c['link']); $course->setCertificate(false); $course->setVerifiedCertificate($c['course-verified']); // Calculate length $length = null; if (!empty($c['course-end'])) { $start = new \DateTime($c['course-start']); $end = new \DateTime($c['course-end']); $length = ceil($start->diff($end)->days / 7); } $course->setLength($length); return $course; }
private function getOnDemandCourse($data = array()) { $dbLanguageMap = $this->dbHelper->getLanguageMap(); $course = new Course(); $course->setShortName(substr('coursera_' . $data['elements'][0]['slug'], 0, 49)); $course->setInitiative($this->initiative); $course->setName($data['elements'][0]['name']); $course->setDescription($data['elements'][0]['description']); $course->setLongDescription(nl2br($data['elements'][0]['description'])); $course->setStream($this->dbHelper->getStreamBySlug('cs')); // Default to Computer Science $course->setUrl('https://www.coursera.org/learn/' . $data['elements'][0]['slug']); $lang = self::$languageMap[$data['elements']['0']['primaryLanguageCodes'][0]]; if (isset($dbLanguageMap[$lang])) { $course->setLanguage($dbLanguageMap[$lang]); } else { $this->out("Language not found " . $data['elements']['0']['primaryLanguageCodes'][0]); } $course->setCertificate(false); $course->setVerifiedCertificate($data['elements'][0]['isVerificationEnabled']); // Add the university foreach ($data['linked']['partners.v1'] as $university) { $ins = new Institution(); $ins->setName($university['name']); $ins->setIsUniversity(true); $ins->setSlug($university['shortName']); $course->addInstitution($this->dbHelper->createInstitutionIfNotExists($ins)); } foreach ($data['linked']['instructors.v1'] as $courseraInstructor) { if (!empty($courseraInstructor['fullName'])) { $insName = $courseraInstructor['fullName']; } else { $insName = $courseraInstructor['firstName'] . ' ' . $courseraInstructor['lastName']; } $course->addInstructor($this->dbHelper->createInstructorIfNotExists($insName)); } // Get Course Details like Syllabus and length $courseDetails = json_decode(file_get_contents(sprintf(self::ONDEMAND_OPENCOURSE_API, $data['elements'][0]['slug'])), true); if (!empty($courseDetails)) { $syllabus = ''; foreach ($courseDetails['courseMaterial']['elements'] as $item) { $syllabus .= "<b>{$item['name']}</b><br/>{$item['description']}<br/><br/>"; } $course->setSyllabus($syllabus); } // Calculate the length of the course $schedule = json_decode(file_get_contents(sprintf(self::ONDEMAND_COURSE_SCHEDULE, $data['elements'][0]['id'])), true); if (!empty($schedule)) { $length = 0; foreach ($schedule['elements'][0]['defaultSchedule']['periods'] as $period) { $length += $period['numberOfWeeks']; } if ($length > 0) { $course->setLength($length); } } return $course; }
private function getCourseEntity($udacityCourse = array()) { $defaultStream = $this->dbHelper->getStreamBySlug('cs'); $langMap = $this->dbHelper->getLanguageMap(); $defaultLanguage = $langMap['English']; $course = new Course(); $course->setShortName(substr('udacity_' . $udacityCourse['slug'], 0, 50)); $course->setInitiative($this->initiative); $course->setName($udacityCourse['title']); $course->setDescription($udacityCourse['short_summary']); $course->setLanguage($defaultLanguage); $course->setStream($defaultStream); // Default to Computer Science $course->setCertificate(false); $course->setUrl($udacityCourse['homepage']); $course->setSyllabus(nl2br($udacityCourse['syllabus'])); $course->setWorkloadMin(6); $course->setWorkloadMax(6); // Calculate length $length = null; $expectedDuration = $udacityCourse['expected_duration']; if ($udacityCourse['expected_duration_unit'] == 'months') { $length = $expectedDuration * 4; } elseif ($udacityCourse['expected_duration_unit'] == 'weeks') { $length = $expectedDuration; } $course->setLength($length); // Calculate Description $course->setLongDescription(nl2br($udacityCourse['summary'] . '<br/><br/><b>Why Take This Course?</b><br/>' . $udacityCourse['expected_learning'])); // Intro Video if (!empty($udacityCourse['teaser_video']['youtube_url'])) { $course->setVideoIntro($udacityCourse['teaser_video']['youtube_url']); } return $course; }
/** * Build a doctrine Course Entity out of a csv row * @param $row * @return Course */ public function getCourseEntity($row) { $course = new Course(); $course->setName($row[0]); $course->setDescription($row[1]); $course->setVideoIntro(str_replace('http', 'https', $row[4])); $course->setUrl($row[5]); $course->setShortName($this->getCourseId($row[5])); $course->setInitiative($this->initiative); // Set the language to arabic $langMap = $this->dbHelper->getLanguageMap(); $course->setLanguage($langMap['Arabic']); // Set the default stream as humanities $defaultStream = $this->dbHelper->getStreamBySlug('humanities'); $course->setStream($defaultStream); // Calculate the length of the course $start = new \DateTime($row[2]); $end = new \DateTime($row[3]); $length = ceil($start->diff($end)->days / 7); $course->setLength($length); return $course; }
public function scrape() { if ($this->isCredential) { $this->scrapeCredentials(); return; } $defaultStream = $this->dbHelper->getStreamBySlug('cs'); $dbLanguageMap = $this->dbHelper->getLanguageMap(); $em = $this->getManager(); $kuber = $this->container->get('kuber'); // File Api $offerings = array(); //$this->buildOnDemandCoursesList(); /************************************* * On Demand Courses *************************************/ $url = 'https://www.coursera.org/api/courses.v1'; $allCourses = json_decode(file_get_contents($url), true); foreach ($allCourses['elements'] as $element) { if ($element['courseType'] == 'v2.ondemand') { $onDemandCourse = json_decode(file_get_contents(sprintf(self::ONDEMAND_COURSE_URL, $element['slug'])), true); //$this->out( $onDemandCourse['elements'][0]['name'] ); if (!$onDemandCourse['elements'][0]['isReal']) { continue; //skip } $c = $this->getOnDemandCourse($onDemandCourse); $dbCourse = null; $dbCourseFromSlug = $this->dbHelper->getCourseByShortName($c->getShortName()); if ($dbCourseFromSlug) { $dbCourse = $dbCourseFromSlug; } else { $dbCourseFromName = $this->findCourseByName($c->getName(), $this->initiative); if ($dbCourseFromName) { $dbCourse = $dbCourseFromName; } } if (empty($dbCourse)) { // Create the course if ($this->doCreate()) { $this->out("NEW COURSE - " . $c->getName()); // NEW COURSE if ($this->doModify()) { $em->persist($c); $em->flush(); if ($onDemandCourse['elements'][0]['promoPhoto']) { $this->uploadImageIfNecessary($onDemandCourse['elements'][0]['promoPhoto'], $c); } // Send an update to Slack $this->dbHelper->sendNewCourseToSlack($c, $this->initiative); // Create an offering $offering = new Offering(); $offering->setShortName($c->getShortName()); $offering->setUrl($c->getUrl()); $offering->setCourse($c); if (isset($onDemandCourse['elements'][0]['plannedLaunchDate'])) { //$this->out( $onDemandCourse['elements'][0]['plannedLaunchDate'] ); try { $startDate = new \DateTime($onDemandCourse['elements'][0]['plannedLaunchDate']); } catch (\Exception $e) { $startDate = new \DateTime(); } $offering->setStatus(Offering::START_MONTH_KNOWN); } else { $startDate = new \DateTime(); $offering->setStatus(Offering::COURSE_OPEN); } $endDate = new \DateTime(); $endDate->add(new \DateInterval("P30D")); $offering->setStartDate($startDate); $offering->setEndDate($endDate); $em->persist($offering); $em->flush(); $this->dbHelper->sendNewOfferingToSlack($offering); } } } else { // Check how many of them are self paced $selfPaced = false; if ($dbCourse->getNextOffering()->getStatus() == Offering::COURSE_OPEN) { $selfPaced = true; } else { if (isset($onDemandCourse['elements'][0]['plannedLaunchDate'])) { $now = new \DateTime(); try { $startDate = new \DateTime($onDemandCourse['elements'][0]['plannedLaunchDate']); } catch (\Exception $e) { $startDate = new \DateTime(); } if ($startDate != $dbCourse->getNextOffering()->getStartDate()) { if ($this->doModify()) { $o = $dbCourse->getNextOffering(); $o->setStartDate($startDate); $o->setStatus(Offering::START_MONTH_KNOWN); $em->persist($o); $em->flush(); $this->out("OnDemand Course Updated Start Date : " . $element['name']); } } else { if ($now >= $dbCourse->getNextOffering()->getStartDate()) { if ($this->doModify()) { //Update the course to be self paced $o = $dbCourse->getNextOffering(); $o->setStatus(Offering::COURSE_OPEN); $em->persist($o); $em->flush(); $this->out("OnDemand Course Updated to Self paced : " . $element['name']); } } } $selfPaced = true; } } if (!$selfPaced) { $this->out("OnDemand Session Missing : " . $element['name']); } } } } exit; /************************************* * Session Based Courses *************************************/ $courseraCourses = $this->getCoursesArray(); foreach ($courseraCourses as $courseraCourse) { $selfServingId = $courseraCourse['self_service_course_id']; $courseraCourseId = $courseraCourse['id']; $courseraCourseShortName = $courseraCourse['short_name']; $courseShortName = 'coursera_' . $courseraCourseShortName; $courseUrl = $this->getCourseLink($courseraCourse); $courseLang = isset(self::$languageMap[$courseraCourse['language']]) ? self::$languageMap[$courseraCourse['language']] : null; $catalogDetails = $this->getDetailsFromCourseraCatalog($courseraCourseId); // Create a course object $course = new Course(); $course->setShortName($courseShortName); $course->setInitiative($this->initiative); $course->setName($courseraCourse['name']); $course->setDescription($courseraCourse['short_description']); $course->setLongDescription($catalogDetails['aboutTheCourse']); $course->setSyllabus($catalogDetails['courseSyllabus']); $course->setStream($defaultStream); // Default to Computer Science $course->setVideoIntro($this->getVideoUrl($courseraCourse)); $course->setUrl($courseUrl); if (isset($dbLanguageMap[$courseLang])) { $course->setLanguage($dbLanguageMap[$courseLang]); } else { $this->out("Language not found " . $courseraCourse['language']); } // Get the workload if (!empty($catalogDetails['estimatedClassWorkload']) && ($workload = $this->getWorkLoad($catalogDetails['estimatedClassWorkload']))) { $course->setWorkloadMin($workload[0]); $course->setWorkloadMax($workload[1]); } // Get the certificate information $sid = $this->getLatestSessionId($catalogDetails); if ($sid) { $sDetails = $this->getDetailsFromSessionCatalog($sid); $course->setCertificate($sDetails['eligibleForCertificates']); $course->setVerifiedCertificate($sDetails['eligibleForSignatureTrack']); } // Add the university foreach ($courseraCourse['universities'] as $university) { $ins = new Institution(); $ins->setName($university['name']); $ins->setIsUniversity(true); $ins->setSlug($university['short_name']); $course->addInstitution($this->dbHelper->createInstitutionIfNotExists($ins)); } // Add categories to search description $searchDesc = array(); foreach ($courseraCourse['categories'] as $category) { $searchDesc[] = $category['name']; } $course->setSearchDesc(implode(' ', $searchDesc)); // Filter out of the offerings to remove those with no status and then get the length of the newest offering $courseraOfferings = array_filter($courseraCourse['courses'], function ($offering) { return !($offering['status'] == 0); }); if (!empty($courseraOfferings)) { $newestOffering = end($courseraOfferings); $course->setLength($this->getOfferingLength($newestOffering['duration_string'])); reset($courseraOfferings); } $courseImage = $courseraCourse['large_icon']; $dbCourse = $this->dbHelper->getCourseByShortName($courseShortName); if (!$dbCourse) { if ($this->doCreate()) { // New course $this->out("NEW COURSE - " . $course->getName()); if ($this->doModify()) { // Get the instructors using the coursera instructor api $courseraInstructors = $this->getInstructorsArray($courseraCourseShortName); foreach ($courseraInstructors as $courseraInstructor) { $insName = $courseraInstructor['first_name'] . ' ' . $courseraInstructor['last_name']; $course->addInstructor($this->dbHelper->createInstructorIfNotExists($insName)); } $em->persist($course); $em->flush(); $this->dbHelper->sendNewCourseToSlack($course, $this->initiative); // Upload the image if ($courseImage) { $this->uploadImageIfNecessary($courseImage, $course); } } } } else { // Check if any fields are modified $courseModified = false; $changedFields = array(); // To keep track of fields that have changed foreach ($this->courseFields as $field) { $getter = 'get' . $field; $setter = 'set' . $field; if ($course->{$getter}() != $dbCourse->{$getter}()) { $courseModified = true; // Add the changed field to the changedFields array $changed = array(); $changed['field'] = $field; $changed['old'] = $dbCourse->{$getter}(); $changed['new'] = $course->{$getter}(); $changedFields[] = $changed; $dbCourse->{$setter}($course->{$getter}()); } } if ($this->doUpdate()) { // Upload the image if ($courseImage) { $this->uploadImageIfNecessary($courseImage, $dbCourse); } } if ($courseModified && $this->doUpdate()) { // Course has been modified $this->out("UPDATE COURSE - " . $dbCourse->getName()); $this->outputChangedFields($changedFields); if ($this->doModify()) { $em->persist($dbCourse); $em->flush(); } } $course = $dbCourse; } // Done with course. Now create offerings foreach ($courseraOfferings as $courseraOffering) { // Create a offering object and set its parameters $offering = new Offering(); $offeringShortName = $courseraCourseShortName . '_' . $courseraCourseId . '_' . $courseraOffering['id']; $offering->setShortName($offeringShortName); $offering->setCourse($course); $offering->setUrl($courseUrl); // Figure out the dates and status $details = array(); $details['status'] = Offering::START_DATES_UNKNOWN; if ($selfServingId == $courseraOffering['id']) { $details['status'] = Offering::COURSE_OPEN; } $details = array_merge($details, $this->getDates($courseraOffering, $this->getOfferingLength($courseraOffering['duration_string']))); $offering->setStartDate(new \DateTime($details['start_date'])); $offering->setStatus($details['status']); if (isset($details['end_date'])) { $offering->setEndDate(new \DateTime($details['end_date'])); } $dbOffering = $this->dbHelper->getOfferingByShortName($offeringShortName); if (!$dbOffering) { if ($this->doCreate()) { $this->out("NEW OFFERING - " . $offering->getName()); if ($this->doModify()) { $em->persist($offering); $em->flush(); } $this->dbHelper->sendNewOfferingToSlack($offering); $offerings[] = $offering; } } else { // old offering. Check if has been modified or not $offeringModified = false; $changedFields = array(); foreach ($this->offeringFields as $field) { $getter = 'get' . $field; $setter = 'set' . $field; if ($offering->{$getter}() != $dbOffering->{$getter}()) { $offeringModified = true; // Add the changed field to the changedFields array $changed = array(); $changed['field'] = $field; $changed['old'] = $dbOffering->{$getter}(); $changed['new'] = $offering->{$getter}(); $changedFields[] = $changed; $dbOffering->{$setter}($offering->{$getter}()); } } if ($offeringModified && $this->doUpdate()) { // Offering has been modified $this->out("UPDATE OFFERING - " . $dbOffering->getName()); $this->outputChangedFields($changedFields); if ($this->doModify()) { $em->persist($dbOffering); $em->flush(); } $offerings[] = $dbOffering; } } } } return $offerings; }
private function getCourseEntity($c = array()) { $defaultStream = $this->dbHelper->getStreamBySlug('cs'); $langMap = $this->dbHelper->getLanguageMap(); $defaultLanguage = $langMap['English']; $course = new Course(); $course->setShortName($c['uuid']); $course->setInitiative($this->initiative); $course->setName($c['name']); $course->setDescription($c['introduction']); $course->setLongDescription($c['description']); $course->setLanguage($defaultLanguage); $course->setStream($defaultStream); // Default to Computer Science $course->setUrl($c['url']); $course->setCertificate($c['has_certificates']); $course->setWorkloadMin($c['hours_per_week']); $course->setWorkloadMax($c['hours_per_week']); // Get the length if ($c['runs']) { $course->setLength($c['runs'][0]['duration_in_weeks']); } return $course; }