public function outOffering(Offering $offering) { // Print the title line $titleLine = $offering->getName(); if ($offering->getStatus() == Offering::START_DATES_KNOWN) { $titleLine .= ' - ' . $offering->getStartDate()->format('M jS'); } $this->output->writeln($titleLine); $this->output->writeln($offering->getUrl()); $this->output->writeln(' '); }
/** * Displays a form to create a new Offering entity. * */ public function newAction(Request $request, $id) { $entity = new Offering(); // Defaults $entity->setStatus(Offering::START_DATES_KNOWN); $startDate = new \DateTime(); //$startDate->sub(new \DateInterval('P60D')); $entity->setStartDate($startDate); $endDate = new \DateTime(); $endDate->add(new \DateInterval('P30D')); $entity->setEndDate($endDate); $em = $this->getDoctrine()->getManager(); // Cloning the entity if ($id) { $type = $request->query->get('type'); if ($type == 'selfpaced') { // Get the course $course = $em->getRepository('ClassCentralSiteBundle:Course')->find($id); $entity->setStatus(Offering::COURSE_OPEN); $entity->setCourse($course); } else { $entity = $em->getRepository('ClassCentralSiteBundle:Offering')->find($id); $entity->setShortName(null); $entity->setUrl(null); } } $form = $this->createForm(new OfferingType(), $entity); return $this->render('ClassCentralSiteBundle:Offering:new.html.twig', array('entity' => $entity, 'form' => $form->createView())); }
public function outOffering(Offering $offering) { // Print the title line $titleLine = $this->getItalics($offering->getName()); if ($offering->getStatus() == Offering::START_DATES_KNOWN) { $titleLine .= ' - ' . $offering->getStartDate()->format('M jS'); } $this->output->writeln($titleLine); // Print out the course length. Exclude Udacity because course length is same if ($offering->getInitiative()->getCode() != 'UDACITY' && $offering->getLength() != 0) { $this->output->writeln($this->getItalics($offering->getLength() . " weeks long")); } // Output the URL $this->output->writeln($offering->getUrl()); // Output an empty line $this->output->writeln(''); }
/** * By default the offering state is upcoming; * @param Offering $offering * @param \DateTime $now * @return int|string */ public static function calculateStateWithDate(Offering $offering, \DateTime $now) { $state = 0; // Ignore offerings that are no longer available if ($offering->getStatus() == Offering::COURSE_NA) { return $state; } $twoWeeksAgo = clone $now; $twoWeeksAgo->sub(new \DateInterval('P14D')); $twoWeeksLater = clone $now; $twoWeeksLater->add(new \DateInterval('P14D')); $yesterday = clone $now; $yesterday->sub(new \DateInterval('P1D')); $startDate = $offering->getStartDate(); $endDate = $offering->getEndDate(); // Check if its recent if ($offering->getStatus() == Offering::START_DATES_KNOWN && $startDate >= $twoWeeksAgo && $startDate <= $twoWeeksLater) { $state += Offering::STATE_RECENT; } // Check if its recently added if ($offering->getCreated() >= $twoWeeksAgo) { $state += Offering::STATE_JUST_ANNOUNCED; } // Check if its self paced if ($offering->getStatus() == Offering::COURSE_OPEN) { $state += Offering::STATE_SELF_PACED; return $state; } // Check if its finished if ($endDate != null && $endDate < $now) { $state += Offering::STATE_FINISHED; return $state; } // Check if its ongoing if ($offering->getStatus() == Offering::START_DATES_KNOWN && $yesterday > $startDate) { $state += Offering::STATE_IN_PROGRESS; return $state; } // If it has reached here it means its upcoming. $state += Offering::STATE_UPCOMING; return $state; }
public function outOffering(Offering $offering) { $course = $offering->getCourse(); $rs = $this->container->get('review'); // Figure out whether the course is new $oneMonthAgo = new \DateTime(); $oneMonthAgo->sub(new \DateInterval("P30D")); $newCourse = false; if ($course->getCreated() >= $oneMonthAgo) { $newCourse = true; } // Is it being offered for he first time if (count($course->getOfferings()) == 1 and $offering->getCreated() > $oneMonthAgo) { $newCourse = true; } if (count($course->getOfferings()) == 1 and $offering->getStatus() != Offering::COURSE_OPEN) { $newCourse = true; } $name = '[' . $offering->getCourse()->getName() . ']' . '(' . $offering->getUrl() . ')'; if ($offering->getInitiative() == null) { $initiative = 'Others'; } else { $initiative = $offering->getInitiative()->getName(); } $startDate = $offering->getDisplayDate(); $startDate = array_shift(explode(',', $startDate)); // Do not show the year to save characters $length = 'NA'; if ($offering->getCourse()->getLength() != 0) { $length = $offering->getCourse()->getLength(); } // Rating $courseRating = round($rs->getRatings($offering->getCourse()->getId()), 1); $courseReviews = $rs->getReviewsArray($offering->getCourse()->getId()); $reviewText = ''; if ($courseRating == 0) { $courseRating = 'NA'; } else { $reviewText = sprintf("(%d)", $courseReviews['count']); } $url = 'https://www.class-central.com' . $this->router->generate('reviews_short_url', array('courseId' => $offering->getCourse()->getId())); //$url .= '#reviews'; $ratingStars = ReviewUtility::getRatingStars($courseRating); if ($courseRating > 0) { $rating = "{$ratingStars} [{$reviewText}]({$url})"; } else { $rating = "{$ratingStars}"; } $new = ''; if ($newCourse) { $new = "[NEW]"; } $this->output->writeln("{$new} {$name} via **{$initiative}**|{$startDate}|{$length}|{$rating}"); }
/** * Get the next offering for this course */ public function getNextOffering() { /** * Filter out offerings which are not available */ $this->getOfferings()->filter(function ($offering) { return $offering->getStatus() == Offering::COURSE_NA; }); $offerings = $this->getOfferings(); if ($offerings->isEmpty()) { //TODO: Handle it correctly // Create a offering $offering = new Offering(); $offering->setCourse($this); $offering->setId(-1); $offering->setUrl($this->getUrl()); $dt = new \DateTime(); $dt->add(new \DateInterval("P1Y")); $offering->setStartDate($dt); $offering->setStatus(Offering::START_DATES_UNKNOWN); return $offering; } $nextOffering = $offerings->first(); $now = new \DateTime(); $upcoming = array(); foreach ($offerings as $offering) { if ($offering->getStartDate() > $now) { $upcoming[] = $offering; } if ($offering->getStartDate() > $nextOffering->getStartDate()) { $nextOffering = $offering; } } if (count($upcoming) > 1) { // Multiple upcoming. Pick the earliest one $nextOffering = array_pop($upcoming); foreach ($upcoming as $offering) { if ($offering->getStartDate() < $nextOffering->getStartDate()) { $nextOffering = $offering; } } } return $nextOffering; }
public function outOffering(Offering $offering) { $formatter = $this->container->get('course_formatter'); echo $formatter->blogFormat($offering->getCourse()) . "\n"; }
public function scrape() { if ($this->isCredential) { $this->scrapeCredentials(); return; } $em = $this->getManager(); $udacityCourses = json_decode(file_get_contents(self::COURSES_API_ENDPOINT), true); $coursesChanged = array(); foreach ($udacityCourses['courses'] as $udacityCourse) { $course = $this->getCourseEntity($udacityCourse); $offering = null; $dbCourse = $this->dbHelper->getCourseByShortName($course->getShortName()); if (!$dbCourse) { $dbCourse = $this->dbHelper->findCourseByName($course->getName(), $this->initiative); } if (!$dbCourse) { // Course does not exist create it. if ($this->doCreate()) { $this->out("NEW COURSE - " . $course->getName()); // NEW COURSE if ($this->doModify()) { $em->persist($course); $em->flush(); $this->dbHelper->sendNewCourseToSlack($course, $this->initiative); if ($udacityCourse['banner_image']) { $this->uploadImageIfNecessary($udacityCourse['banner_image'], $course); } // Create new offering $offering = new Offering(); $offering->setCourse($course); $offering->setUrl($course->getUrl()); $startDate = new \DateTime(); $offering->setStartDate($startDate); $endDate = new \DateTime(); $endDate->add(new \DateInterval('P30D')); $offering->setEndDate($endDate); $offering->setStatus(Offering::COURSE_OPEN); $em->persist($offering); $em->flush(); } $courseChanged = true; } } else { // Check if any fields are modified $courseModified = false; $changedFields = array(); // To keep track of fields that have changed foreach ($this->courseFields as $field) { $getter = 'get' . $field; $setter = 'set' . $field; if ($course->{$getter}() != $dbCourse->{$getter}()) { $courseModified = true; // Add the changed field to the changedFields array $changed = array(); $changed['field'] = $field; $changed['old'] = $dbCourse->{$getter}(); $changed['new'] = $course->{$getter}(); $changedFields[] = $changed; $dbCourse->{$setter}($course->{$getter}()); } } if ($courseModified && $this->doUpdate()) { //$this->out( "Database course changed " . $dbCourse->getName()); // Course has been modified $this->out("UPDATE COURSE - " . $dbCourse->getName() . " - " . $dbCourse->getId()); $this->outputChangedFields($changedFields); if ($this->doModify()) { $em->persist($dbCourse); $em->flush(); if ($udacityCourse['banner_image']) { $this->uploadImageIfNecessary($udacityCourse['banner_image'], $dbCourse); } } $courseChanged = true; } // Check if offering has been modified $offering = $dbCourse->getNextOffering(); if ($offering->getUrl() != $course->getUrl() && $this->doUpdate()) { $this->out("UPDATE COURSE - " . $dbCourse->getName() . " - " . $dbCourse->getId()); // Offering modified $this->outputChangedFields(array(array('field' => 'offering Url', 'old' => $offering->getUrl(), 'new' => $course->getUrl()))); if ($this->doModify()) { $offering->setUrl($course->getUrl()); $em->persist($offering); $em->flush(); } $courseChanged = true; } $course = $dbCourse; } } if ($courseChanged) { $coursesChanged[] = $course; } return $coursesChanged; }
private function getOfferingEntity($run, $course) { $offering = new Offering(); $offering->setShortName($run['uuid']); $offering->setCourse($course); $offering->setUrl($course->getUrl()); if ($run['start_date']) { $startDate = new \DateTime($run['start_date']); $endDate = new \DateTime($run['start_date']); $days = $run['duration_in_weeks'] * 7; $endDate->add(new \DateInterval("P{$days}D")); $offering->setStartDate($startDate); $offering->setEndDate($endDate); $offering->setStatus(Offering::START_DATES_KNOWN); } else { $curYear = date('Y'); $startDate = new \DateTime("{$curYear}-12-30"); // Dec 30 $endDate = new \DateTime("{$curYear}-12-31"); // Dec 31 $offering->setStartDate($startDate); $offering->setEndDate($endDate); $offering->setStatus(Offering::START_YEAR_KNOWN); } return $offering; }
/** * Using the CSV */ public function scrape() { $this->buildSelfPacedCourseList(); $tagService = $this->container->get('tag'); // Get the course list from the new RSS API $edxCourses = file_get_contents(self::EDX_RSS_API_JSON); $edxCourses = json_decode($edxCourses, true); foreach ($edxCourses['value']['items'] as $edxCourse) { $em = $this->getManager(); $course = $this->getCourseEntity($edxCourse); $cTags = array(); if (is_array($edxCourse['course:school'])) { foreach ($edxCourse['course:school'] as $school) { $cTags[] = strtolower($school); } } else { $cTags[] = strtolower($edxCourse['course:school']); } $dbCourse = $this->dbHelper->getCourseByShortName($course->getShortName()); // Do a fuzzy match on the course title if (!$dbCourse) { $result = $this->findCourseByName($edxCourse['title'], $this->initiative); if (count($result) > 1) { $this->out("DUPLICATE ENTRIES FOR: " . $edxCourse['title']); foreach ($result as $item) { $this->out("COURSE ID" . $item->getId()); } continue; } else { if (count($result) == 1) { $dbCourse = $result; } } } if (!$dbCourse) { if ($this->doCreate()) { $this->out("NEW COURSE - " . $course->getName()); // NEW COURSE if ($this->doModify()) { $em->persist($course); $em->flush(); $tagService->saveCourseTags($course, $cTags); $this->dbHelper->sendNewCourseToSlack($course, $this->initiative); if ($edxCourse['course:image-banner']) { $this->uploadImageIfNecessary($edxCourse['course:image-banner'], $course); } } } } else { // Check if any fields are modified $courseModified = false; $changedFields = array(); // To keep track of fields that have changed foreach ($this->courseFields as $field) { $getter = 'get' . $field; $setter = 'set' . $field; if ($course->{$getter}() != $dbCourse->{$getter}()) { $courseModified = true; // Add the changed field to the changedFields array $changed = array(); $changed['field'] = $field; $changed['old'] = $dbCourse->{$getter}(); $changed['new'] = $course->{$getter}(); $changedFields[] = $changed; $dbCourse->{$setter}($course->{$getter}()); } } if ($courseModified && $this->doUpdate()) { //$this->out( "Database course changed " . $dbCourse->getName()); // Course has been modified $this->out("UPDATE COURSE - " . $dbCourse->getName() . " - " . $dbCourse->getId()); $this->outputChangedFields($changedFields); if ($this->doModify()) { $em->persist($dbCourse); $em->flush(); // Update tags $tagService->saveCourseTags($dbCourse, $cTags); if ($edxCourse['course:image-banner']) { $this->uploadImageIfNecessary($edxCourse['course:image-banner'], $dbCourse); } } } $course = $dbCourse; } /*************************** * CREATE OR UPDATE OFFERING ***************************/ $offering = new Offering(); $osn = $this->getOfferingShortName($edxCourse); $offering->setShortName($osn); $offering->setCourse($course); $offering->setUrl($edxCourse['link']); $offering->setStatus(Offering::START_DATES_KNOWN); $offering->setStartDate(new \DateTime($edxCourse['course:start'])); if (empty($edxCourse['course:end'])) { // Put an end date for 4 weeks in the future $endDate = new \DateTime($edxCourse['course:start']); $endDate->add(new \DateInterval("P30D")); } else { $endDate = new \DateTime($edxCourse['course:end']); } $offering->setEndDate($endDate); $dbOffering = $this->dbHelper->getOfferingByShortName($osn); if (!$dbOffering) { if ($this->doCreate()) { $this->out("NEW OFFERING - " . $offering->getName()); if ($this->doModify()) { $em->persist($offering); $em->flush(); } $this->dbHelper->sendNewOfferingToSlack($offering); $offerings[] = $offering; } } else { // old offering. Check if has been modified or not $offeringModified = false; $changedFields = array(); foreach ($this->offeringFields as $field) { $getter = 'get' . $field; $setter = 'set' . $field; if ($offering->{$getter}() != $dbOffering->{$getter}()) { $offeringModified = true; // Add the changed field to the changedFields array $changed = array(); $changed['field'] = $field; $changed['old'] = $dbOffering->{$getter}(); $changed['new'] = $offering->{$getter}(); $changedFields[] = $changed; $dbOffering->{$setter}($offering->{$getter}()); } } if ($offeringModified && $this->doUpdate()) { // Offering has been modified $this->out("UPDATE OFFERING - " . $dbOffering->getName()); $this->outputChangedFields($changedFields); if ($this->doModify()) { $em->persist($dbOffering); $em->flush(); } $offerings[] = $dbOffering; } } } return $offerings; }
public function scrape() { if ($this->isCredential) { $this->scrapeCredentials(); return; } $defaultStream = $this->dbHelper->getStreamBySlug('cs'); $dbLanguageMap = $this->dbHelper->getLanguageMap(); $em = $this->getManager(); $kuber = $this->container->get('kuber'); // File Api $offerings = array(); //$this->buildOnDemandCoursesList(); /************************************* * On Demand Courses *************************************/ $url = 'https://www.coursera.org/api/courses.v1'; $allCourses = json_decode(file_get_contents($url), true); foreach ($allCourses['elements'] as $element) { if ($element['courseType'] == 'v2.ondemand') { $onDemandCourse = json_decode(file_get_contents(sprintf(self::ONDEMAND_COURSE_URL, $element['slug'])), true); //$this->out( $onDemandCourse['elements'][0]['name'] ); if (!$onDemandCourse['elements'][0]['isReal']) { continue; //skip } $c = $this->getOnDemandCourse($onDemandCourse); $dbCourse = null; $dbCourseFromSlug = $this->dbHelper->getCourseByShortName($c->getShortName()); if ($dbCourseFromSlug) { $dbCourse = $dbCourseFromSlug; } else { $dbCourseFromName = $this->findCourseByName($c->getName(), $this->initiative); if ($dbCourseFromName) { $dbCourse = $dbCourseFromName; } } if (empty($dbCourse)) { // Create the course if ($this->doCreate()) { $this->out("NEW COURSE - " . $c->getName()); // NEW COURSE if ($this->doModify()) { $em->persist($c); $em->flush(); if ($onDemandCourse['elements'][0]['promoPhoto']) { $this->uploadImageIfNecessary($onDemandCourse['elements'][0]['promoPhoto'], $c); } // Send an update to Slack $this->dbHelper->sendNewCourseToSlack($c, $this->initiative); $dbCourse = $c; } } } else { // Update the course details $changedFields = $this->dbHelper->changedFields($this->onDemandCourseFields, $c, $dbCourse); if (!empty($changedFields) && $this->doUpdate()) { $this->out("UPDATE COURSE - " . $dbCourse->getName()); $this->outputChangedFields($changedFields); if ($this->doModify()) { $em->persist($dbCourse); $em->flush(); $this->uploadImageIfNecessary($onDemandCourse['elements'][0]['promoPhoto'], $dbCourse); } } // Check how many of them are self paced $selfPaced = false; if ($dbCourse->getNextOffering()->getStatus() == Offering::COURSE_OPEN) { $selfPaced = true; } else { /* if( isset($onDemandCourse['elements'][0]['plannedLaunchDate'])) { $now = new \DateTime(); try{ $startDate = new \DateTime( $onDemandCourse['elements'][0]['plannedLaunchDate'] ); } catch(\Exception $e) { $startDate = new \DateTime(); } if( $startDate != $dbCourse->getNextOffering()->getStartDate() ) { if ($this->doModify()) { $o = $dbCourse->getNextOffering(); $o->setStartDate( $startDate ); $o->setStatus( Offering::START_MONTH_KNOWN ); $em->persist( $o ); $em->flush(); $this->out("OnDemand Course Updated Start Date : " . $element['name']) ; } } else if ( $now >= $dbCourse->getNextOffering()->getStartDate() ) { if ($this->doModify()) { //Update the course to be self paced $o = $dbCourse->getNextOffering(); $o->setStatus( Offering::COURSE_OPEN ); $em->persist( $o ); $em->flush(); $this->out("OnDemand Course Updated to Self paced : " . $element['name']) ; } } $selfPaced = true; } */ } // Update the sessions. $courseId = $onDemandCourse['elements'][0]['id']; $sessionDetails = json_decode(file_get_contents(sprintf(self::ONDEMAND_SESSION_IDS, $courseId)), true); if (empty($sessionDetails['elements'])) { // Create an offering $offering = new Offering(); $offering->setShortName($dbCourse->getShortName()); $offering->setUrl($dbCourse->getUrl()); $offering->setCourse($dbCourse); if (isset($onDemandCourse['elements'][0]['plannedLaunchDate'])) { try { // Self paced Not Started - But will Start in the future $this->out("SELF PACED FUTURE COURSE : " . $dbCourse->getName()); $startDate = new \DateTime($onDemandCourse['elements'][0]['plannedLaunchDate']); $endDate = new \DateTime($onDemandCourse['elements'][0]['plannedLaunchDate']); $endDate->add(new \DateInterval("P30D")); $offering->setStatus(Offering::START_DATES_KNOWN); } catch (\Exception $e) { continue; } } else { // Self paced course that can be accessed right now $this->out("SELF PACED COURSE : " . $dbCourse->getName()); $startDate = new \DateTime(); $offering->setStatus(Offering::COURSE_OPEN); $endDate = new \DateTime(); $endDate->add(new \DateInterval("P30D")); if ($dbCourse->getNextOffering()->getStatus() == Offering::COURSE_OPEN) { // Already self paced nothing to be done here continue; } } $offering->setStartDate($startDate); $offering->setEndDate($endDate); // Check if offering exists $dbOffering = $this->dbHelper->getOfferingByShortName($dbCourse->getShortName()); if ($dbOffering) { // Check if the dates and other details are right $this->offeringChangedFields($offering, $dbOffering); } else { // Save and Create the offering if ($this->doCreate()) { $this->out("NEW OFFERING - " . $offering->getName()); if ($this->doModify()) { $em->persist($offering); $em->flush(); $this->dbHelper->sendNewOfferingToSlack($offering); } } } } else { $dbOffering = null; // Regularly Scheduled Course $this->out("Regularly Scheduled Course : " . $dbCourse->getName()); foreach ($dbCourse->getOfferings() as $o) { if ($o->getShortName() == $dbCourse->getShortName()) { $dbOffering = $o; // A course with future announced date becomes current and has sessions break; } } foreach ($sessionDetails['elements'] as $session) { $sessionId = $session['id']; $offeringShortName = 'coursera_' . $sessionId; // Create an offering $offering = new Offering(); $offering->setShortName($offeringShortName); $offering->setUrl($dbCourse->getUrl()); $offering->setCourse($dbCourse); $offering->setStatus(Offering::START_DATES_KNOWN); $startDate = new \DateTime('@' . intval($session['startedAt'] / 1000)); $endDate = new \DateTime('@' . intval($session['endedAt'] / 1000)); $startDate->setTimezone(new \DateTimeZone('America/Los_Angeles')); $endDate->setTimezone(new \DateTimeZone('America/Los_Angeles')); $offering->setStartDate($startDate); $offering->setEndDate($endDate); // Check if offering exists if (!$dbOffering) { $dbOffering = $this->dbHelper->getOfferingByShortName($offeringShortName); } if ($dbOffering) { // Check if the dates and other details are right $this->offeringChangedFields($offering, $dbOffering); } else { if ($this->doCreate()) { $this->out("NEW OFFERING - " . $offering->getName()); if ($this->doModify()) { $em->persist($offering); $em->flush(); $this->dbHelper->sendNewOfferingToSlack($offering); } } } $dbOffering = null; } } if (!$selfPaced) { //$this->out("OnDemand Session Missing : " . $element['name']) ; } } } } /************************************* * Session Based Courses *************************************/ $courseraCourses = $this->getCoursesArray(); foreach ($courseraCourses as $courseraCourse) { $selfServingId = $courseraCourse['self_service_course_id']; $courseraCourseId = $courseraCourse['id']; $courseraCourseShortName = $courseraCourse['short_name']; $courseShortName = 'coursera_' . $courseraCourseShortName; $courseUrl = $this->getCourseLink($courseraCourse); $courseLang = isset(self::$languageMap[$courseraCourse['language']]) ? self::$languageMap[$courseraCourse['language']] : null; $catalogDetails = $this->getDetailsFromCourseraCatalog($courseraCourseId); // Create a course object $course = new Course(); $course->setShortName($courseShortName); $course->setInitiative($this->initiative); $course->setName($courseraCourse['name']); $course->setDescription($courseraCourse['short_description']); $course->setLongDescription($catalogDetails['aboutTheCourse']); $course->setSyllabus($catalogDetails['courseSyllabus']); $course->setStream($defaultStream); // Default to Computer Science $course->setVideoIntro($this->getVideoUrl($courseraCourse)); $course->setUrl($courseUrl); if (isset($dbLanguageMap[$courseLang])) { $course->setLanguage($dbLanguageMap[$courseLang]); } else { $this->out("Language not found " . $courseraCourse['language']); } // Get the workload if (!empty($catalogDetails['estimatedClassWorkload']) && ($workload = $this->getWorkLoad($catalogDetails['estimatedClassWorkload']))) { $course->setWorkloadMin($workload[0]); $course->setWorkloadMax($workload[1]); } // Get the certificate information $sid = $this->getLatestSessionId($catalogDetails); if ($sid) { $sDetails = $this->getDetailsFromSessionCatalog($sid); $course->setCertificate($sDetails['eligibleForCertificates']); $course->setVerifiedCertificate($sDetails['eligibleForSignatureTrack']); } // Add the university foreach ($courseraCourse['universities'] as $university) { $ins = new Institution(); $ins->setName($university['name']); $ins->setIsUniversity(true); $ins->setSlug($university['short_name']); $course->addInstitution($this->dbHelper->createInstitutionIfNotExists($ins)); } // Add categories to search description $searchDesc = array(); foreach ($courseraCourse['categories'] as $category) { $searchDesc[] = $category['name']; } $course->setSearchDesc(implode(' ', $searchDesc)); // Filter out of the offerings to remove those with no status and then get the length of the newest offering $courseraOfferings = array_filter($courseraCourse['courses'], function ($offering) { return !($offering['status'] == 0); }); if (!empty($courseraOfferings)) { $newestOffering = end($courseraOfferings); $course->setLength($this->getOfferingLength($newestOffering['duration_string'])); reset($courseraOfferings); } $courseImage = $courseraCourse['large_icon']; $dbCourse = $this->dbHelper->getCourseByShortName($courseShortName); if (!$dbCourse) { if ($this->doCreate()) { // New course $this->out("NEW COURSE - " . $course->getName()); if ($this->doModify()) { // Get the instructors using the coursera instructor api $courseraInstructors = $this->getInstructorsArray($courseraCourseShortName); foreach ($courseraInstructors as $courseraInstructor) { $insName = $courseraInstructor['first_name'] . ' ' . $courseraInstructor['last_name']; $course->addInstructor($this->dbHelper->createInstructorIfNotExists($insName)); } $em->persist($course); $em->flush(); $this->dbHelper->sendNewCourseToSlack($course, $this->initiative); // Upload the image if ($courseImage) { $this->uploadImageIfNecessary($courseImage, $course); } } } } else { // Check if any fields are modified $courseModified = false; $changedFields = array(); // To keep track of fields that have changed foreach ($this->courseFields as $field) { $getter = 'get' . $field; $setter = 'set' . $field; if ($course->{$getter}() != $dbCourse->{$getter}()) { $courseModified = true; // Add the changed field to the changedFields array $changed = array(); $changed['field'] = $field; $changed['old'] = $dbCourse->{$getter}(); $changed['new'] = $course->{$getter}(); $changedFields[] = $changed; $dbCourse->{$setter}($course->{$getter}()); } } if ($this->doUpdate()) { // Upload the image if ($courseImage) { $this->uploadImageIfNecessary($courseImage, $dbCourse); } } if ($courseModified && $this->doUpdate()) { // Course has been modified $this->out("UPDATE COURSE - " . $dbCourse->getName()); $this->outputChangedFields($changedFields); if ($this->doModify()) { $em->persist($dbCourse); $em->flush(); } } $course = $dbCourse; } // Done with course. Now create offerings foreach ($courseraOfferings as $courseraOffering) { // Create a offering object and set its parameters $offering = new Offering(); $offeringShortName = $courseraCourseShortName . '_' . $courseraCourseId . '_' . $courseraOffering['id']; $offering->setShortName($offeringShortName); $offering->setCourse($course); $offering->setUrl($courseUrl); // Figure out the dates and status $details = array(); $details['status'] = Offering::START_DATES_UNKNOWN; if ($selfServingId == $courseraOffering['id']) { $details['status'] = Offering::COURSE_OPEN; } $details = array_merge($details, $this->getDates($courseraOffering, $this->getOfferingLength($courseraOffering['duration_string']))); $offering->setStartDate(new \DateTime($details['start_date'])); $offering->setStatus($details['status']); if (isset($details['end_date'])) { $offering->setEndDate(new \DateTime($details['end_date'])); } $dbOffering = $this->dbHelper->getOfferingByShortName($offeringShortName); if (!$dbOffering) { if ($this->doCreate()) { $this->out("NEW OFFERING - " . $offering->getName()); if ($this->doModify()) { $em->persist($offering); $em->flush(); } $this->dbHelper->sendNewOfferingToSlack($offering); $offerings[] = $offering; } } else { // old offering. Check if has been modified or not $offeringModified = false; $changedFields = array(); foreach ($this->offeringFields as $field) { $getter = 'get' . $field; $setter = 'set' . $field; if ($offering->{$getter}() != $dbOffering->{$getter}()) { $offeringModified = true; // Add the changed field to the changedFields array $changed = array(); $changed['field'] = $field; $changed['old'] = $dbOffering->{$getter}(); $changed['new'] = $offering->{$getter}(); $changedFields[] = $changed; $dbOffering->{$setter}($offering->{$getter}()); } } if ($offeringModified && $this->doUpdate()) { // Offering has been modified $this->out("UPDATE OFFERING - " . $dbOffering->getName()); $this->outputChangedFields($changedFields); if ($this->doModify()) { $em->persist($dbOffering); $em->flush(); } $offerings[] = $dbOffering; } } } } return $offerings; }
private function buildOffering($id, $date, $state) { $o = new Offering(); $o->setId($id); $o->setStartDate($date); // End date $endDate = clone $date; $endDate->add(new \DateInterval("P1M")); $o->setEndDate($endDate); $o->setStatus($state); return $o; }
public function buildForm(FormBuilder $builder, array $options) { $builder->add('startDate')->add('endDate')->add('status', 'choice', array('choices' => \ClassCentral\SiteBundle\Entity\Offering::getStatuses()))->add('course', null, array('property' => 'name'))->add('name')->add('initiative', null, array('required' => false, 'empty_value' => true))->add('url')->add('videoIntro')->add('length')->add('instructors', null, array('required' => false, 'empty_value' => true)); }
private function getOfferingEntity($canvasCourse, Course $course) { $offering = new Offering(); $offering->setCourse($course); $offering->setUrl($course->getUrl()); $offering->setShortName('canvas_' . $canvasCourse['id']); if ($canvasCourse['date'] == 'Self-paced') { $startDate = new \DateTime(); $endDate = new \DateTime(); $endDate->add(new \DateInterval('P30D')); $offering->setStatus(Offering::COURSE_OPEN); $offering->setStartDate($startDate); $offering->setEndDate($endDate); } elseif (strpos($canvasCourse['date'], 'Start') === false) { // Date is of the follow format: Jan 25 - Feb 29, 2016 $date = explode(',', $canvasCourse['date']); $year = $date[1]; $day = explode('-', $date[0]); $startDate = new \DateTime($day[0] . $year); $endDate = new \DateTime($day[1] . ' ' . $year); $offering->setStatus(Offering::START_DATES_KNOWN); $offering->setStartDate($startDate); $offering->setEndDate($endDate); } else { // Date contains Started ... $date = substr($canvasCourse['date'], strpos($canvasCourse['date'], ' ')); $startDate = new \DateTime($date); $endDate = new \DateTime('2018-12-31'); $offering->setStatus(Offering::COURSE_OPEN); $offering->setStartDate($startDate); $offering->setEndDate($endDate); $this->out($offering->getDisplayDate()); } return $offering; }
public function buildForm(FormBuilderInterface $builder, array $options) { $builder->add('startDate')->add('endDate')->add('status', 'choice', array('choices' => \ClassCentral\SiteBundle\Entity\Offering::getStatuses()))->add('course', null, array('property' => 'name'))->add('shortName', null, array('required' => false))->add('url'); }
public function outOffering(Offering $offering) { $rs = $this->container->get('review'); $name = '[' . $offering->getCourse()->getName() . ']' . '(' . $offering->getUrl() . ')'; if ($offering->getInitiative() == null) { $initiative = 'Others'; } else { $initiative = $offering->getInitiative()->getName(); } $startDate = $offering->getDisplayDate(); $startDate = array_shift(explode(',', $startDate)); // Do not show the year to save characters $length = 'NA'; if ($offering->getCourse()->getLength() != 0) { $length = $offering->getCourse()->getLength(); } // Rating $courseRating = round($rs->getRatings($offering->getCourse()->getId()), 1); $courseReviews = $rs->getReviewsArray($offering->getCourse()->getId()); $reviewText = ''; if ($courseRating == 0) { $courseRating = 'NA'; } else { $reviewText = sprintf("(%d)", $courseReviews['count']); } $url = 'https://www.class-central.com' . $this->router->generate('reviews_short_url', array('courseId' => $offering->getCourse()->getId())); $url .= '#reviews'; $ratingStars = ReviewUtility::getRatingStars($courseRating); if ($courseRating > 0) { $rating = "{$ratingStars} [{$reviewText}]({$url})"; } else { $rating = "{$ratingStars}"; } $this->output->writeln("{$name} via **{$initiative}**|{$startDate}|{$length}|{$rating}"); }
public function outOffering(Offering $offering) { $rs = $this->container->get('review'); $name = '[' . $offering->getCourse()->getName() . ']' . '(' . $offering->getUrl() . ')'; if ($offering->getInitiative() == null) { $initiative = 'Others'; } else { $initiative = $offering->getInitiative()->getName(); } $startDate = $offering->getDisplayDate(); $length = 'NA'; if ($offering->getCourse()->getLength() != 0) { $length = $offering->getCourse()->getLength(); } // Rating $courseRating = round($rs->getRatings($offering->getCourse()->getId()), 1); $courseReviews = $rs->getReviewsArray($offering->getCourse()->getId()); $reviewText = ''; if ($courseRating == 0) { $courseRating = 'NA'; } else { $reviewText = sprintf("(%d %s)", $courseReviews['count'], $courseReviews['count'] == 1 ? 'review' : 'reviews'); } $url = 'https://www.class-central.com' . $this->router->generate('ClassCentralSiteBundle_mooc', array('id' => $offering->getCourse()->getId(), 'slug' => $offering->getCourse()->getSlug())); $url .= '#reviews'; $ratingStars = ReviewUtility::getRatingStars($courseRating); $rating = "{$ratingStars} [{$reviewText}]({$url})"; $this->output->writeln("{$name}|{$startDate}|{$length}|{$initiative}|{$rating}"); }
/** * Returns an array of values for a particular display * @param \ClassCentral\SiteBundle\Entity\Offering $offering * @return Array */ public function getOfferingArray(Offering $offering) { $offeringArray = array(); $course = $offering->getCourse(); $offeringArray['id'] = $offering->getId(); $offeringArray['name'] = $offering->getName(); $offeringArray['url'] = $offering->getUrl(); $offeringArray['videoIntro'] = $course->getVideoIntro(); $offeringArray['length'] = $course->getLength(); $offeringArray['startTimeStamp'] = $offering->getStartTimestamp(); $offeringArray['displayDate'] = $offering->getDisplayDate(); $offeringArray['startDate'] = $offering->getStartDate()->format('d-m-Y'); $offeringArray['microdataDate'] = $offering->getMicrodataDate(); $offeringArray['status'] = $offering->getStatus(); // Stream $stream = $course->getStream(); $offeringArray['stream']['name'] = $stream->getName(); $offeringArray['stream']['slug'] = $stream->getSlug(); $offeringArray['stream']['showInNav'] = $stream->getShowInNav(); $initiative = $course->getInitiative(); $offeringArray['initiative']['name'] = ''; if ($initiative != null) { $offeringArray['initiative']['name'] = $initiative->getName(); $offeringArray['initiative']['url'] = $initiative->getUrl(); $offeringArray['initiative']['tooltip'] = $initiative->getTooltip(); $offeringArray['initiative']['code'] = strtolower($initiative->getCode()); } // Language $language = $course->getLanguage(); $offeringArray['language']['name'] = ''; if ($language) { $offeringArray['language']['name'] = $language->getName(); } // Add Institutions $offeringArray['institutions'] = array(); foreach ($course->getInstitutions() as $institution) { $offeringArray['institutions'][] = array('name' => $institution->getName(), 'url' => $institution->getUrl(), 'slug' => $institution->getSlug(), 'isUniversity' => $institution->getIsUniversity()); } $offeringArray['instructors'] = array(); foreach ($course->getInstructors() as $instructor) { $offeringArray['instructors'][] = $instructor->getName(); } // Generate the course url $offeringArray['courseSlug'] = $course->getSlug(); $offeringArray['courseId'] = $course->getId(); return $offeringArray; }
public function sendNewOfferingToSlack(Offering $offering) { try { $course = $offering->getCourse(); $initiative = $offering->getInitiative(); $providerInfo = PageHeaderFactory::get($initiative); $coursePageUrl = $this->scraper->getContainer()->getParameter('baseurl') . $this->scraper->getContainer()->get('router')->generate('ClassCentralSiteBundle_mooc', array('id' => $course->getId(), 'slug' => $course->getSlug())); $logo = $this->scraper->getContainer()->getParameter('rackspace_cdn_base_url') . $providerInfo->getImageUrl(); $message = "[New Session] *{$course->getName()}* - {$offering->getDisplayDate()}\n" . $coursePageUrl; $this->scraper->getContainer()->get('slack_client')->to('#cc-activity-data')->from($initiative->getName())->withIcon($logo)->send($message); } catch (\Exception $e) { } }
public function scrape() { $em = $this->getManager(); // Array of offerings created or updated $offerings = array(); $this->out("Scraping " . $this->initiative->getName()); // Step 1: Getting a list of course URLs $this->out("Getting a list of course pages"); $urls = $this->getListOfCoursePages(); $urlsCount = count($urls); // Step 2: Go through the page and create/update offering $this->out("Number of courses found: {$urlsCount}"); $this->out("Gathering details about each course"); $courseDetails = array(); foreach ($urls as $url) { if (!$url) { continue; } $courseDetail = array(); $this->domParser->load(file_get_contents(self::BASE_URL . $url)); // Ignore self paced if (!$this->domParser->find('h2.offering_dates_date', 0)) { continue; } // Get Name and shortName $nameString = $this->domParser->find('h1.page-title', 0)->plaintext; $openBracketPosition = strpos($nameString, '('); $closeBracketPosition = strpos($nameString, ')'); $courseDetail['name'] = substr($nameString, 0, $openBracketPosition - 1); $courseDetail['shortName'] = substr($nameString, $openBracketPosition + 1, $closeBracketPosition - $openBracketPosition - 1); if ($courseDetail['name'] == 'Introduction to Nursing in Healthcar') { $courseDetail['name'] = 'Introduction to Nursing in Healthcare'; $courseDetail['shortName'] = 'IntroNur'; } // Get the video id from the url // eg. www.youtube.com/embed/Bw8HkjGQb3U?wmode=opaque&rel=0&showinfo=0 $youtubeIdPosition = 31; $video = 'http://' . $this->domParser->find('iframe.media-youtube-player', 0)->src; $questionMarkPosition = strpos($video, '?'); $courseDetail['video'] = 'http://www.youtube.com/watch?v=' . substr($video, $youtubeIdPosition, $questionMarkPosition - $youtubeIdPosition); $instructors = trim($this->domParser->find('div[id=subject-teacher-tagline]', 0)->plaintext); // Remove the 'by' $instructors = substr($instructors, 3); $courseDetail['instructors'] = explode(' & ', $instructors); $courseDetail['desc'] = $this->domParser->find('div.offering_body', 0)->plaintext; $courseDetail['start_date'] = $this->domParser->find('h2.offering_dates_date', 0)->plaintext; $courseDetail['end_date'] = $this->domParser->find('h2.offering_dates_date', 1)->plaintext; $courseDetail['url'] = $url; print_r($courseDetail); $courseDetails[] = $courseDetail; $this->domParser->clear(); } $this->out(count($courseDetails) . ' course pages found'); // Default stream $stream = $this->dbHelper->getStreamBySlug('business'); $this->out("Default stream is " . $stream->getName()); foreach ($courseDetails as $courseDetail) { /** * Taking a shortcut here. Check if a course is created or not. If it isn't create the * course,offering, etc. Updates are ignored * TODO: Not take a shortcut */ // Build a course object $course = new Course(); $courseShortName = 'open2study_' . $courseDetail['shortName']; $course->setShortName($courseShortName); $course->setInitiative($this->initiative); $course->setName($courseDetail['name']); $course->setDescription($courseDetail['desc']); $course->setStream($stream); // Default to Business $course->setVideoIntro($courseDetail['video']); $course->setUrl(self::BASE_URL . $courseDetail['url']); $dbCourse = $this->dbHelper->getCourseByShortName($courseShortName); if (!$dbCourse) { if ($this->doCreate()) { // New course $this->out("NEW COURSE - " . $course->getName()); if ($this->doModify()) { foreach ($courseDetail['instructors'] as $instructor) { $course->addInstructor($this->dbHelper->createInstructorIfNotExists($instructor)); } $em->persist($course); $em->flush(); } } } else { $course = $dbCourse; } // Check if offering exists $shortName = $this->getOfferingShortName($courseDetail); $offering = $this->dbHelper->getOfferingByShortName($shortName); if ($offering) { continue; } // Check if create offering is oon if (!$this->doCreate()) { $offerings[] = $offering; // Add it to the offerings table continue; } $offering = new Offering(); $offering->setCourse($course); $offering->setStartDate(\DateTime::createFromFormat("d/m/Y", $courseDetail['start_date'])); $offering->setEndDate(\DateTime::createFromFormat("d/m/Y", $courseDetail['end_date'])); $offering->setStatus(Offering::START_DATES_KNOWN); $offering->setLength(4); $offering->setShortName($shortName); $offering->setUrl(self::BASE_URL . $courseDetail['url']); $offering->setVideoIntro($courseDetail['video']); $offering->setSearchDesc($courseDetail['desc']); $offering->setCreated(new \DateTime()); if ($this->doModify()) { try { $em->persist($offering); $em->flush(); $this->out("OFFERING {$courseDetail['name']} created"); } catch (\Exception $e) { $this->out("OFFERING {$courseDetail['name']} creation FAILED"); } } $offerings[] = $offering; } return $offerings; }
/** * Using the CSV */ public function scrape() { if ($this->isCredential) { $this->scrapeCredentials(); return; } //$this->buildSelfPacedCourseList(); $tagService = $this->container->get('tag'); // Get the course list from the new RSS API $page = 0; /* $fp = fopen("extras/edX_prices.csv", "w"); fputcsv($fp, array( 'Course Name', 'Certificate Name', 'Prices(in $)' )); */ while (true) { $page++; $this->out("Retrieving PAGE #" . $page); $edxCourses = file_get_contents(sprintf(self::EDX_RSS_API, $page)); $edxCourses = str_replace("course:", "course-", $edxCourses); $edxCourses = str_replace("staff:", "staff-", $edxCourses); $simpleXml = simplexml_load_string($edxCourses); $edxCourses = json_encode($simpleXml); $edxCourses = json_decode($edxCourses, true); if (empty($edxCourses['channel']['item'])) { break; } foreach ($edxCourses['channel']['item'] as $edxCourse) { $em = $this->getManager(); $course = $this->getCourseEntity($edxCourse); $cTags = array(); if (is_array($edxCourse['course-school'])) { foreach ($edxCourse['course-school'] as $school) { $cTags[] = strtolower($school); } } else { $cTags[] = strtolower($edxCourse['course-school']); } $courseId = $edxCourse['course-id']; /** $fileName = "/tmp/edx/{$edxCourse['title']}.json"; if( file_exists($fileName) ) { $productPrices = json_decode(file_get_contents($fileName),true); } else { $productPrices = json_decode(file_get_contents( sprintf(self::EDX_ENROLLMENT_COURSE_DETAIL, $courseId) ),true); if(!empty($productPrices)) { file_put_contents($fileName, json_encode($productPrices)); } } foreach( $productPrices['course_modes'] as $mode) { if($mode['name'] == 'Honor Certificate' || $mode['name'] == 'Audit') { continue; } $this->out( $edxCourse['title'] . '|||' . $mode['name'] . '|||' . $mode['min_price']); fputcsv($fp, array( $edxCourse['title'], $mode['name'], $mode['min_price'] )); } continue; * */ $dbCourse = $this->dbHelper->getCourseByShortName($course->getShortName()); // Do a fuzzy match on the course title if (!$dbCourse) { $result = $this->findCourseByName($edxCourse['title'], $this->initiative); if (count($result) > 1) { $this->out("DUPLICATE ENTRIES FOR: " . $edxCourse['title']); foreach ($result as $item) { $this->out("COURSE ID" . $item->getId()); } continue; } else { if (count($result) == 1) { $dbCourse = $result; } } } if (!$dbCourse) { if ($this->doCreate()) { $this->out("NEW COURSE - " . $course->getName()); // NEW COURSE if ($this->doModify()) { // Add instructors if (!empty($edxCourse['course-instructors']['course-staff']['staff-name'])) { print_r($edxCourse['course-instructors']['course-staff']['staff-name']); $insName = $edxCourse['course-instructors']['course-staff']['staff-name']; $course->addInstructor($this->dbHelper->createInstructorIfNotExists($insName)); } elseif (!empty($edxCourse['course-instructors']['course-staff'])) { foreach ($edxCourse['course-instructors']['course-staff'] as $staff) { $insName = $staff['staff-name']; $course->addInstructor($this->dbHelper->createInstructorIfNotExists($insName)); } } $em->persist($course); $em->flush(); $tagService->saveCourseTags($course, $cTags); $this->dbHelper->sendNewCourseToSlack($course, $this->initiative); if ($edxCourse['course-image-banner']) { $this->uploadImageIfNecessary($edxCourse['course-image-banner'], $course); } } } } else { // Check if any fields are modified $courseModified = false; $changedFields = array(); // To keep track of fields that have changed foreach ($this->courseFields as $field) { $getter = 'get' . $field; $setter = 'set' . $field; if ($course->{$getter}() != $dbCourse->{$getter}()) { $courseModified = true; // Add the changed field to the changedFields array $changed = array(); $changed['field'] = $field; $changed['old'] = $dbCourse->{$getter}(); $changed['new'] = $course->{$getter}(); $changedFields[] = $changed; $dbCourse->{$setter}($course->{$getter}()); } } if ($courseModified && $this->doUpdate()) { //$this->out( "Database course changed " . $dbCourse->getName()); // Course has been modified $this->out("UPDATE COURSE - " . $dbCourse->getName() . " - " . $dbCourse->getId()); $this->outputChangedFields($changedFields); if ($this->doModify()) { $em->persist($dbCourse); $em->flush(); // Update tags $tagService->saveCourseTags($dbCourse, $cTags); if ($edxCourse['course-image-banner']) { $this->uploadImageIfNecessary($edxCourse['course-image-banner'], $dbCourse); } } } $course = $dbCourse; } /*************************** * CREATE OR UPDATE OFFERING ***************************/ $offering = new Offering(); $osn = $this->getOfferingShortName($edxCourse); $offering->setShortName($osn); $offering->setCourse($course); $offering->setUrl($edxCourse['link']); $offering->setStatus(Offering::START_DATES_KNOWN); $offering->setStartDate(new \DateTime($edxCourse['course-start'])); if (empty($edxCourse['course-end'])) { // Put an end date for 4 weeks in the future $endDate = new \DateTime($edxCourse['course-start']); $endDate->add(new \DateInterval("P30D")); } else { $endDate = new \DateTime($edxCourse['course-end']); } $offering->setEndDate($endDate); $dbOffering = $this->dbHelper->getOfferingByShortName($osn); if (!$dbOffering) { if ($this->doCreate()) { $this->out("NEW OFFERING - " . $offering->getName()); if ($this->doModify()) { $em->persist($offering); $em->flush(); } $this->dbHelper->sendNewOfferingToSlack($offering); $offerings[] = $offering; } } else { // old offering. Check if has been modified or not $offeringModified = false; $changedFields = array(); foreach ($this->offeringFields as $field) { $getter = 'get' . $field; $setter = 'set' . $field; if ($offering->{$getter}() != $dbOffering->{$getter}()) { $offeringModified = true; // Add the changed field to the changedFields array $changed = array(); $changed['field'] = $field; $changed['old'] = $dbOffering->{$getter}(); $changed['new'] = $offering->{$getter}(); $changedFields[] = $changed; $dbOffering->{$setter}($offering->{$getter}()); } } if ($offeringModified && $this->doUpdate()) { // Offering has been modified $this->out("UPDATE OFFERING - " . $dbOffering->getName()); $this->outputChangedFields($changedFields); if ($this->doModify()) { $em->persist($dbOffering); $em->flush(); } $offerings[] = $dbOffering; } } } } /** fclose($fp); */ return $offerings; }
/** * Build a doctrine Offering Entity out of a csv row * @param $row * @return Offering */ public function getOfferingEntity($row) { $offering = new Offering(); $offering->setShortName($this->getOfferingId($row[5])); $offering->setStartDate(new \DateTime($row[2])); $offering->setEndDate(new \DateTime($row[3])); $offering->setStatus(Offering::START_DATES_KNOWN); $offering->setUrl($row[5]); return $offering; }