/**
* Displays a form to create a new Offering entity.
*
*/
public function newAction(Request $request, $id)
{
$entity = new Offering();
// Defaults
$entity->setStatus(Offering::START_DATES_KNOWN);
$startDate = new \DateTime();
//$startDate->sub(new \DateInterval('P60D'));
$entity->setStartDate($startDate);
$endDate = new \DateTime();
$endDate->add(new \DateInterval('P30D'));
$entity->setEndDate($endDate);
$em = $this->getDoctrine()->getManager();
// Cloning the entity
if ($id) {
$type = $request->query->get('type');
if ($type == 'selfpaced') {
// Get the course
$course = $em->getRepository('ClassCentralSiteBundle:Course')->find($id);
$entity->setStatus(Offering::COURSE_OPEN);
$entity->setCourse($course);
} else {
$entity = $em->getRepository('ClassCentralSiteBundle:Offering')->find($id);
$entity->setShortName(null);
$entity->setUrl(null);
}
}
$form = $this->createForm(new OfferingType(), $entity);
return $this->render('ClassCentralSiteBundle:Offering:new.html.twig', array('entity' => $entity, 'form' => $form->createView()));
}
/**
* Using the CSV
*/
public function scrape()
{
if ($this->isCredential) {
$this->scrapeCredentials();
return;
}
//$this->buildSelfPacedCourseList();
$tagService = $this->container->get('tag');
// Get the course list from the new RSS API
$page = 0;
/*
$fp = fopen("extras/edX_prices.csv", "w");
fputcsv($fp, array(
'Course Name', 'Certificate Name', 'Prices(in $)'
));
*/
while (true) {
$page++;
$this->out("Retrieving PAGE #" . $page);
$edxCourses = file_get_contents(sprintf(self::EDX_RSS_API, $page));
$edxCourses = str_replace("course:", "course-", $edxCourses);
$edxCourses = str_replace("staff:", "staff-", $edxCourses);
$simpleXml = simplexml_load_string($edxCourses);
$edxCourses = json_encode($simpleXml);
$edxCourses = json_decode($edxCourses, true);
if (empty($edxCourses['channel']['item'])) {
break;
}
foreach ($edxCourses['channel']['item'] as $edxCourse) {
$em = $this->getManager();
$course = $this->getCourseEntity($edxCourse);
$cTags = array();
if (is_array($edxCourse['course-school'])) {
foreach ($edxCourse['course-school'] as $school) {
$cTags[] = strtolower($school);
}
} else {
$cTags[] = strtolower($edxCourse['course-school']);
}
$courseId = $edxCourse['course-id'];
/**
$fileName = "/tmp/edx/{$edxCourse['title']}.json";
if( file_exists($fileName) )
{
$productPrices = json_decode(file_get_contents($fileName),true);
}
else
{
$productPrices = json_decode(file_get_contents( sprintf(self::EDX_ENROLLMENT_COURSE_DETAIL, $courseId) ),true);
if(!empty($productPrices))
{
file_put_contents($fileName, json_encode($productPrices));
}
}
foreach( $productPrices['course_modes'] as $mode)
{
if($mode['name'] == 'Honor Certificate' || $mode['name'] == 'Audit')
{
continue;
}
$this->out( $edxCourse['title'] . '|||' . $mode['name'] . '|||' . $mode['min_price']);
fputcsv($fp, array(
$edxCourse['title'], $mode['name'], $mode['min_price']
));
}
continue;
* */
$dbCourse = $this->dbHelper->getCourseByShortName($course->getShortName());
// Do a fuzzy match on the course title
if (!$dbCourse) {
$result = $this->findCourseByName($edxCourse['title'], $this->initiative);
if (count($result) > 1) {
$this->out("DUPLICATE ENTRIES FOR: " . $edxCourse['title']);
foreach ($result as $item) {
$this->out("COURSE ID" . $item->getId());
}
continue;
} else {
if (count($result) == 1) {
$dbCourse = $result;
}
}
}
if (!$dbCourse) {
if ($this->doCreate()) {
$this->out("NEW COURSE - " . $course->getName());
// NEW COURSE
if ($this->doModify()) {
// Add instructors
if (!empty($edxCourse['course-instructors']['course-staff']['staff-name'])) {
print_r($edxCourse['course-instructors']['course-staff']['staff-name']);
$insName = $edxCourse['course-instructors']['course-staff']['staff-name'];
$course->addInstructor($this->dbHelper->createInstructorIfNotExists($insName));
} elseif (!empty($edxCourse['course-instructors']['course-staff'])) {
foreach ($edxCourse['course-instructors']['course-staff'] as $staff) {
$insName = $staff['staff-name'];
$course->addInstructor($this->dbHelper->createInstructorIfNotExists($insName));
}
}
$em->persist($course);
$em->flush();
$tagService->saveCourseTags($course, $cTags);
$this->dbHelper->sendNewCourseToSlack($course, $this->initiative);
if ($edxCourse['course-image-banner']) {
$this->uploadImageIfNecessary($edxCourse['course-image-banner'], $course);
}
}
}
} else {
// Check if any fields are modified
$courseModified = false;
$changedFields = array();
// To keep track of fields that have changed
foreach ($this->courseFields as $field) {
$getter = 'get' . $field;
$setter = 'set' . $field;
if ($course->{$getter}() != $dbCourse->{$getter}()) {
$courseModified = true;
// Add the changed field to the changedFields array
$changed = array();
$changed['field'] = $field;
$changed['old'] = $dbCourse->{$getter}();
$changed['new'] = $course->{$getter}();
$changedFields[] = $changed;
$dbCourse->{$setter}($course->{$getter}());
}
}
if ($courseModified && $this->doUpdate()) {
//$this->out( "Database course changed " . $dbCourse->getName());
// Course has been modified
$this->out("UPDATE COURSE - " . $dbCourse->getName() . " - " . $dbCourse->getId());
$this->outputChangedFields($changedFields);
if ($this->doModify()) {
$em->persist($dbCourse);
$em->flush();
// Update tags
$tagService->saveCourseTags($dbCourse, $cTags);
if ($edxCourse['course-image-banner']) {
$this->uploadImageIfNecessary($edxCourse['course-image-banner'], $dbCourse);
}
}
}
$course = $dbCourse;
}
/***************************
* CREATE OR UPDATE OFFERING
***************************/
$offering = new Offering();
$osn = $this->getOfferingShortName($edxCourse);
$offering->setShortName($osn);
$offering->setCourse($course);
$offering->setUrl($edxCourse['link']);
$offering->setStatus(Offering::START_DATES_KNOWN);
$offering->setStartDate(new \DateTime($edxCourse['course-start']));
if (empty($edxCourse['course-end'])) {
// Put an end date for 4 weeks in the future
$endDate = new \DateTime($edxCourse['course-start']);
$endDate->add(new \DateInterval("P30D"));
} else {
$endDate = new \DateTime($edxCourse['course-end']);
}
$offering->setEndDate($endDate);
$dbOffering = $this->dbHelper->getOfferingByShortName($osn);
if (!$dbOffering) {
if ($this->doCreate()) {
$this->out("NEW OFFERING - " . $offering->getName());
if ($this->doModify()) {
$em->persist($offering);
$em->flush();
}
$this->dbHelper->sendNewOfferingToSlack($offering);
$offerings[] = $offering;
}
} else {
// old offering. Check if has been modified or not
$offeringModified = false;
$changedFields = array();
foreach ($this->offeringFields as $field) {
$getter = 'get' . $field;
$setter = 'set' . $field;
if ($offering->{$getter}() != $dbOffering->{$getter}()) {
$offeringModified = true;
// Add the changed field to the changedFields array
$changed = array();
$changed['field'] = $field;
$changed['old'] = $dbOffering->{$getter}();
$changed['new'] = $offering->{$getter}();
$changedFields[] = $changed;
$dbOffering->{$setter}($offering->{$getter}());
}
}
if ($offeringModified && $this->doUpdate()) {
// Offering has been modified
$this->out("UPDATE OFFERING - " . $dbOffering->getName());
$this->outputChangedFields($changedFields);
if ($this->doModify()) {
$em->persist($dbOffering);
$em->flush();
}
$offerings[] = $dbOffering;
}
}
}
}
/**
fclose($fp);
*/
return $offerings;
}
/**
* Using the CSV
*/
public function scrape()
{
$this->buildSelfPacedCourseList();
$tagService = $this->container->get('tag');
// Get the course list from the new RSS API
$edxCourses = file_get_contents(self::EDX_RSS_API_JSON);
$edxCourses = json_decode($edxCourses, true);
foreach ($edxCourses['value']['items'] as $edxCourse) {
$em = $this->getManager();
$course = $this->getCourseEntity($edxCourse);
$cTags = array();
if (is_array($edxCourse['course:school'])) {
foreach ($edxCourse['course:school'] as $school) {
$cTags[] = strtolower($school);
}
} else {
$cTags[] = strtolower($edxCourse['course:school']);
}
$dbCourse = $this->dbHelper->getCourseByShortName($course->getShortName());
// Do a fuzzy match on the course title
if (!$dbCourse) {
$result = $this->findCourseByName($edxCourse['title'], $this->initiative);
if (count($result) > 1) {
$this->out("DUPLICATE ENTRIES FOR: " . $edxCourse['title']);
foreach ($result as $item) {
$this->out("COURSE ID" . $item->getId());
}
continue;
} else {
if (count($result) == 1) {
$dbCourse = $result;
}
}
}
if (!$dbCourse) {
if ($this->doCreate()) {
$this->out("NEW COURSE - " . $course->getName());
// NEW COURSE
if ($this->doModify()) {
$em->persist($course);
$em->flush();
$tagService->saveCourseTags($course, $cTags);
$this->dbHelper->sendNewCourseToSlack($course, $this->initiative);
if ($edxCourse['course:image-banner']) {
$this->uploadImageIfNecessary($edxCourse['course:image-banner'], $course);
}
}
}
} else {
// Check if any fields are modified
$courseModified = false;
$changedFields = array();
// To keep track of fields that have changed
foreach ($this->courseFields as $field) {
$getter = 'get' . $field;
$setter = 'set' . $field;
if ($course->{$getter}() != $dbCourse->{$getter}()) {
$courseModified = true;
// Add the changed field to the changedFields array
$changed = array();
$changed['field'] = $field;
$changed['old'] = $dbCourse->{$getter}();
$changed['new'] = $course->{$getter}();
$changedFields[] = $changed;
$dbCourse->{$setter}($course->{$getter}());
}
}
if ($courseModified && $this->doUpdate()) {
//$this->out( "Database course changed " . $dbCourse->getName());
// Course has been modified
$this->out("UPDATE COURSE - " . $dbCourse->getName() . " - " . $dbCourse->getId());
$this->outputChangedFields($changedFields);
if ($this->doModify()) {
$em->persist($dbCourse);
$em->flush();
// Update tags
$tagService->saveCourseTags($dbCourse, $cTags);
if ($edxCourse['course:image-banner']) {
$this->uploadImageIfNecessary($edxCourse['course:image-banner'], $dbCourse);
}
}
}
$course = $dbCourse;
}
/***************************
* CREATE OR UPDATE OFFERING
***************************/
$offering = new Offering();
$osn = $this->getOfferingShortName($edxCourse);
$offering->setShortName($osn);
$offering->setCourse($course);
$offering->setUrl($edxCourse['link']);
$offering->setStatus(Offering::START_DATES_KNOWN);
$offering->setStartDate(new \DateTime($edxCourse['course:start']));
if (empty($edxCourse['course:end'])) {
// Put an end date for 4 weeks in the future
$endDate = new \DateTime($edxCourse['course:start']);
$endDate->add(new \DateInterval("P30D"));
} else {
$endDate = new \DateTime($edxCourse['course:end']);
}
$offering->setEndDate($endDate);
$dbOffering = $this->dbHelper->getOfferingByShortName($osn);
if (!$dbOffering) {
if ($this->doCreate()) {
$this->out("NEW OFFERING - " . $offering->getName());
if ($this->doModify()) {
$em->persist($offering);
$em->flush();
}
$this->dbHelper->sendNewOfferingToSlack($offering);
$offerings[] = $offering;
}
} else {
// old offering. Check if has been modified or not
$offeringModified = false;
$changedFields = array();
foreach ($this->offeringFields as $field) {
$getter = 'get' . $field;
$setter = 'set' . $field;
if ($offering->{$getter}() != $dbOffering->{$getter}()) {
$offeringModified = true;
// Add the changed field to the changedFields array
$changed = array();
$changed['field'] = $field;
$changed['old'] = $dbOffering->{$getter}();
$changed['new'] = $offering->{$getter}();
$changedFields[] = $changed;
$dbOffering->{$setter}($offering->{$getter}());
}
}
if ($offeringModified && $this->doUpdate()) {
// Offering has been modified
$this->out("UPDATE OFFERING - " . $dbOffering->getName());
$this->outputChangedFields($changedFields);
if ($this->doModify()) {
$em->persist($dbOffering);
$em->flush();
}
$offerings[] = $dbOffering;
}
}
}
return $offerings;
}
public function scrape()
{
if ($this->isCredential) {
$this->scrapeCredentials();
return;
}
$defaultStream = $this->dbHelper->getStreamBySlug('cs');
$dbLanguageMap = $this->dbHelper->getLanguageMap();
$em = $this->getManager();
$kuber = $this->container->get('kuber');
// File Api
$offerings = array();
//$this->buildOnDemandCoursesList();
/*************************************
* On Demand Courses
*************************************/
$url = 'https://www.coursera.org/api/courses.v1';
$allCourses = json_decode(file_get_contents($url), true);
foreach ($allCourses['elements'] as $element) {
if ($element['courseType'] == 'v2.ondemand') {
$onDemandCourse = json_decode(file_get_contents(sprintf(self::ONDEMAND_COURSE_URL, $element['slug'])), true);
//$this->out( $onDemandCourse['elements'][0]['name'] );
if (!$onDemandCourse['elements'][0]['isReal']) {
continue;
//skip
}
$c = $this->getOnDemandCourse($onDemandCourse);
$dbCourse = null;
$dbCourseFromSlug = $this->dbHelper->getCourseByShortName($c->getShortName());
if ($dbCourseFromSlug) {
$dbCourse = $dbCourseFromSlug;
} else {
$dbCourseFromName = $this->findCourseByName($c->getName(), $this->initiative);
if ($dbCourseFromName) {
$dbCourse = $dbCourseFromName;
}
}
if (empty($dbCourse)) {
// Create the course
if ($this->doCreate()) {
$this->out("NEW COURSE - " . $c->getName());
// NEW COURSE
if ($this->doModify()) {
$em->persist($c);
$em->flush();
if ($onDemandCourse['elements'][0]['promoPhoto']) {
$this->uploadImageIfNecessary($onDemandCourse['elements'][0]['promoPhoto'], $c);
}
// Send an update to Slack
$this->dbHelper->sendNewCourseToSlack($c, $this->initiative);
$dbCourse = $c;
}
}
} else {
// Update the course details
$changedFields = $this->dbHelper->changedFields($this->onDemandCourseFields, $c, $dbCourse);
if (!empty($changedFields) && $this->doUpdate()) {
$this->out("UPDATE COURSE - " . $dbCourse->getName());
$this->outputChangedFields($changedFields);
if ($this->doModify()) {
$em->persist($dbCourse);
$em->flush();
$this->uploadImageIfNecessary($onDemandCourse['elements'][0]['promoPhoto'], $dbCourse);
}
}
// Check how many of them are self paced
$selfPaced = false;
if ($dbCourse->getNextOffering()->getStatus() == Offering::COURSE_OPEN) {
$selfPaced = true;
} else {
/*
if( isset($onDemandCourse['elements'][0]['plannedLaunchDate']))
{
$now = new \DateTime();
try{
$startDate = new \DateTime( $onDemandCourse['elements'][0]['plannedLaunchDate'] );
}
catch(\Exception $e)
{
$startDate = new \DateTime();
}
if( $startDate != $dbCourse->getNextOffering()->getStartDate() )
{
if ($this->doModify())
{
$o = $dbCourse->getNextOffering();
$o->setStartDate( $startDate );
$o->setStatus( Offering::START_MONTH_KNOWN );
$em->persist( $o );
$em->flush();
$this->out("OnDemand Course Updated Start Date : " . $element['name']) ;
}
}
else if ( $now >= $dbCourse->getNextOffering()->getStartDate() )
{
if ($this->doModify())
{
//Update the course to be self paced
$o = $dbCourse->getNextOffering();
$o->setStatus( Offering::COURSE_OPEN );
$em->persist( $o );
$em->flush();
$this->out("OnDemand Course Updated to Self paced : " . $element['name']) ; }
}
$selfPaced = true;
}
*/
}
// Update the sessions.
$courseId = $onDemandCourse['elements'][0]['id'];
$sessionDetails = json_decode(file_get_contents(sprintf(self::ONDEMAND_SESSION_IDS, $courseId)), true);
if (empty($sessionDetails['elements'])) {
// Create an offering
$offering = new Offering();
$offering->setShortName($dbCourse->getShortName());
$offering->setUrl($dbCourse->getUrl());
$offering->setCourse($dbCourse);
if (isset($onDemandCourse['elements'][0]['plannedLaunchDate'])) {
try {
// Self paced Not Started - But will Start in the future
$this->out("SELF PACED FUTURE COURSE : " . $dbCourse->getName());
$startDate = new \DateTime($onDemandCourse['elements'][0]['plannedLaunchDate']);
$endDate = new \DateTime($onDemandCourse['elements'][0]['plannedLaunchDate']);
$endDate->add(new \DateInterval("P30D"));
$offering->setStatus(Offering::START_DATES_KNOWN);
} catch (\Exception $e) {
continue;
}
} else {
// Self paced course that can be accessed right now
$this->out("SELF PACED COURSE : " . $dbCourse->getName());
$startDate = new \DateTime();
$offering->setStatus(Offering::COURSE_OPEN);
$endDate = new \DateTime();
$endDate->add(new \DateInterval("P30D"));
if ($dbCourse->getNextOffering()->getStatus() == Offering::COURSE_OPEN) {
// Already self paced nothing to be done here
continue;
}
}
$offering->setStartDate($startDate);
$offering->setEndDate($endDate);
// Check if offering exists
$dbOffering = $this->dbHelper->getOfferingByShortName($dbCourse->getShortName());
if ($dbOffering) {
// Check if the dates and other details are right
$this->offeringChangedFields($offering, $dbOffering);
} else {
// Save and Create the offering
if ($this->doCreate()) {
$this->out("NEW OFFERING - " . $offering->getName());
if ($this->doModify()) {
$em->persist($offering);
$em->flush();
$this->dbHelper->sendNewOfferingToSlack($offering);
}
}
}
} else {
$dbOffering = null;
// Regularly Scheduled Course
$this->out("Regularly Scheduled Course : " . $dbCourse->getName());
foreach ($dbCourse->getOfferings() as $o) {
if ($o->getShortName() == $dbCourse->getShortName()) {
$dbOffering = $o;
// A course with future announced date becomes current and has sessions
break;
}
}
foreach ($sessionDetails['elements'] as $session) {
$sessionId = $session['id'];
$offeringShortName = 'coursera_' . $sessionId;
// Create an offering
$offering = new Offering();
$offering->setShortName($offeringShortName);
$offering->setUrl($dbCourse->getUrl());
$offering->setCourse($dbCourse);
$offering->setStatus(Offering::START_DATES_KNOWN);
$startDate = new \DateTime('@' . intval($session['startedAt'] / 1000));
$endDate = new \DateTime('@' . intval($session['endedAt'] / 1000));
$startDate->setTimezone(new \DateTimeZone('America/Los_Angeles'));
$endDate->setTimezone(new \DateTimeZone('America/Los_Angeles'));
$offering->setStartDate($startDate);
$offering->setEndDate($endDate);
// Check if offering exists
if (!$dbOffering) {
$dbOffering = $this->dbHelper->getOfferingByShortName($offeringShortName);
}
if ($dbOffering) {
// Check if the dates and other details are right
$this->offeringChangedFields($offering, $dbOffering);
} else {
if ($this->doCreate()) {
$this->out("NEW OFFERING - " . $offering->getName());
if ($this->doModify()) {
$em->persist($offering);
$em->flush();
$this->dbHelper->sendNewOfferingToSlack($offering);
}
}
}
$dbOffering = null;
}
}
if (!$selfPaced) {
//$this->out("OnDemand Session Missing : " . $element['name']) ;
}
}
}
}
/*************************************
* Session Based Courses
*************************************/
$courseraCourses = $this->getCoursesArray();
foreach ($courseraCourses as $courseraCourse) {
$selfServingId = $courseraCourse['self_service_course_id'];
$courseraCourseId = $courseraCourse['id'];
$courseraCourseShortName = $courseraCourse['short_name'];
$courseShortName = 'coursera_' . $courseraCourseShortName;
$courseUrl = $this->getCourseLink($courseraCourse);
$courseLang = isset(self::$languageMap[$courseraCourse['language']]) ? self::$languageMap[$courseraCourse['language']] : null;
$catalogDetails = $this->getDetailsFromCourseraCatalog($courseraCourseId);
// Create a course object
$course = new Course();
$course->setShortName($courseShortName);
$course->setInitiative($this->initiative);
$course->setName($courseraCourse['name']);
$course->setDescription($courseraCourse['short_description']);
$course->setLongDescription($catalogDetails['aboutTheCourse']);
$course->setSyllabus($catalogDetails['courseSyllabus']);
$course->setStream($defaultStream);
// Default to Computer Science
$course->setVideoIntro($this->getVideoUrl($courseraCourse));
$course->setUrl($courseUrl);
if (isset($dbLanguageMap[$courseLang])) {
$course->setLanguage($dbLanguageMap[$courseLang]);
} else {
$this->out("Language not found " . $courseraCourse['language']);
}
// Get the workload
if (!empty($catalogDetails['estimatedClassWorkload']) && ($workload = $this->getWorkLoad($catalogDetails['estimatedClassWorkload']))) {
$course->setWorkloadMin($workload[0]);
$course->setWorkloadMax($workload[1]);
}
// Get the certificate information
$sid = $this->getLatestSessionId($catalogDetails);
if ($sid) {
$sDetails = $this->getDetailsFromSessionCatalog($sid);
$course->setCertificate($sDetails['eligibleForCertificates']);
$course->setVerifiedCertificate($sDetails['eligibleForSignatureTrack']);
}
// Add the university
foreach ($courseraCourse['universities'] as $university) {
$ins = new Institution();
$ins->setName($university['name']);
$ins->setIsUniversity(true);
$ins->setSlug($university['short_name']);
$course->addInstitution($this->dbHelper->createInstitutionIfNotExists($ins));
}
// Add categories to search description
$searchDesc = array();
foreach ($courseraCourse['categories'] as $category) {
$searchDesc[] = $category['name'];
}
$course->setSearchDesc(implode(' ', $searchDesc));
// Filter out of the offerings to remove those with no status and then get the length of the newest offering
$courseraOfferings = array_filter($courseraCourse['courses'], function ($offering) {
return !($offering['status'] == 0);
});
if (!empty($courseraOfferings)) {
$newestOffering = end($courseraOfferings);
$course->setLength($this->getOfferingLength($newestOffering['duration_string']));
reset($courseraOfferings);
}
$courseImage = $courseraCourse['large_icon'];
$dbCourse = $this->dbHelper->getCourseByShortName($courseShortName);
if (!$dbCourse) {
if ($this->doCreate()) {
// New course
$this->out("NEW COURSE - " . $course->getName());
if ($this->doModify()) {
// Get the instructors using the coursera instructor api
$courseraInstructors = $this->getInstructorsArray($courseraCourseShortName);
foreach ($courseraInstructors as $courseraInstructor) {
$insName = $courseraInstructor['first_name'] . ' ' . $courseraInstructor['last_name'];
$course->addInstructor($this->dbHelper->createInstructorIfNotExists($insName));
}
$em->persist($course);
$em->flush();
$this->dbHelper->sendNewCourseToSlack($course, $this->initiative);
// Upload the image
if ($courseImage) {
$this->uploadImageIfNecessary($courseImage, $course);
}
}
}
} else {
// Check if any fields are modified
$courseModified = false;
$changedFields = array();
// To keep track of fields that have changed
foreach ($this->courseFields as $field) {
$getter = 'get' . $field;
$setter = 'set' . $field;
if ($course->{$getter}() != $dbCourse->{$getter}()) {
$courseModified = true;
// Add the changed field to the changedFields array
$changed = array();
$changed['field'] = $field;
$changed['old'] = $dbCourse->{$getter}();
$changed['new'] = $course->{$getter}();
$changedFields[] = $changed;
$dbCourse->{$setter}($course->{$getter}());
}
}
if ($this->doUpdate()) {
// Upload the image
if ($courseImage) {
$this->uploadImageIfNecessary($courseImage, $dbCourse);
}
}
if ($courseModified && $this->doUpdate()) {
// Course has been modified
$this->out("UPDATE COURSE - " . $dbCourse->getName());
$this->outputChangedFields($changedFields);
if ($this->doModify()) {
$em->persist($dbCourse);
$em->flush();
}
}
$course = $dbCourse;
}
// Done with course. Now create offerings
foreach ($courseraOfferings as $courseraOffering) {
// Create a offering object and set its parameters
$offering = new Offering();
$offeringShortName = $courseraCourseShortName . '_' . $courseraCourseId . '_' . $courseraOffering['id'];
$offering->setShortName($offeringShortName);
$offering->setCourse($course);
$offering->setUrl($courseUrl);
// Figure out the dates and status
$details = array();
$details['status'] = Offering::START_DATES_UNKNOWN;
if ($selfServingId == $courseraOffering['id']) {
$details['status'] = Offering::COURSE_OPEN;
}
$details = array_merge($details, $this->getDates($courseraOffering, $this->getOfferingLength($courseraOffering['duration_string'])));
$offering->setStartDate(new \DateTime($details['start_date']));
$offering->setStatus($details['status']);
if (isset($details['end_date'])) {
$offering->setEndDate(new \DateTime($details['end_date']));
}
$dbOffering = $this->dbHelper->getOfferingByShortName($offeringShortName);
if (!$dbOffering) {
if ($this->doCreate()) {
$this->out("NEW OFFERING - " . $offering->getName());
if ($this->doModify()) {
$em->persist($offering);
$em->flush();
}
$this->dbHelper->sendNewOfferingToSlack($offering);
$offerings[] = $offering;
}
} else {
// old offering. Check if has been modified or not
$offeringModified = false;
$changedFields = array();
foreach ($this->offeringFields as $field) {
$getter = 'get' . $field;
$setter = 'set' . $field;
if ($offering->{$getter}() != $dbOffering->{$getter}()) {
$offeringModified = true;
// Add the changed field to the changedFields array
$changed = array();
$changed['field'] = $field;
$changed['old'] = $dbOffering->{$getter}();
$changed['new'] = $offering->{$getter}();
$changedFields[] = $changed;
$dbOffering->{$setter}($offering->{$getter}());
}
}
if ($offeringModified && $this->doUpdate()) {
// Offering has been modified
$this->out("UPDATE OFFERING - " . $dbOffering->getName());
$this->outputChangedFields($changedFields);
if ($this->doModify()) {
$em->persist($dbOffering);
$em->flush();
}
$offerings[] = $dbOffering;
}
}
}
}
return $offerings;
}
/**
* Get the next offering for this course
*/
public function getNextOffering()
{
/**
* Filter out offerings which are not available
*/
$this->getOfferings()->filter(function ($offering) {
return $offering->getStatus() == Offering::COURSE_NA;
});
$offerings = $this->getOfferings();
if ($offerings->isEmpty()) {
//TODO: Handle it correctly
// Create a offering
$offering = new Offering();
$offering->setCourse($this);
$offering->setId(-1);
$offering->setUrl($this->getUrl());
$dt = new \DateTime();
$dt->add(new \DateInterval("P1Y"));
$offering->setStartDate($dt);
$offering->setStatus(Offering::START_DATES_UNKNOWN);
return $offering;
}
$nextOffering = $offerings->first();
$now = new \DateTime();
$upcoming = array();
foreach ($offerings as $offering) {
if ($offering->getStartDate() > $now) {
$upcoming[] = $offering;
}
if ($offering->getStartDate() > $nextOffering->getStartDate()) {
$nextOffering = $offering;
}
}
if (count($upcoming) > 1) {
// Multiple upcoming. Pick the earliest one
$nextOffering = array_pop($upcoming);
foreach ($upcoming as $offering) {
if ($offering->getStartDate() < $nextOffering->getStartDate()) {
$nextOffering = $offering;
}
}
}
return $nextOffering;
}
public function scrape()
{
if ($this->isCredential) {
$this->scrapeCredentials();
return;
}
$em = $this->getManager();
$udacityCourses = json_decode(file_get_contents(self::COURSES_API_ENDPOINT), true);
$coursesChanged = array();
foreach ($udacityCourses['courses'] as $udacityCourse) {
$course = $this->getCourseEntity($udacityCourse);
$offering = null;
$dbCourse = $this->dbHelper->getCourseByShortName($course->getShortName());
if (!$dbCourse) {
$dbCourse = $this->dbHelper->findCourseByName($course->getName(), $this->initiative);
}
if (!$dbCourse) {
// Course does not exist create it.
if ($this->doCreate()) {
$this->out("NEW COURSE - " . $course->getName());
// NEW COURSE
if ($this->doModify()) {
$em->persist($course);
$em->flush();
$this->dbHelper->sendNewCourseToSlack($course, $this->initiative);
if ($udacityCourse['banner_image']) {
$this->uploadImageIfNecessary($udacityCourse['banner_image'], $course);
}
// Create new offering
$offering = new Offering();
$offering->setCourse($course);
$offering->setUrl($course->getUrl());
$startDate = new \DateTime();
$offering->setStartDate($startDate);
$endDate = new \DateTime();
$endDate->add(new \DateInterval('P30D'));
$offering->setEndDate($endDate);
$offering->setStatus(Offering::COURSE_OPEN);
$em->persist($offering);
$em->flush();
}
$courseChanged = true;
}
} else {
// Check if any fields are modified
$courseModified = false;
$changedFields = array();
// To keep track of fields that have changed
foreach ($this->courseFields as $field) {
$getter = 'get' . $field;
$setter = 'set' . $field;
if ($course->{$getter}() != $dbCourse->{$getter}()) {
$courseModified = true;
// Add the changed field to the changedFields array
$changed = array();
$changed['field'] = $field;
$changed['old'] = $dbCourse->{$getter}();
$changed['new'] = $course->{$getter}();
$changedFields[] = $changed;
$dbCourse->{$setter}($course->{$getter}());
}
}
if ($courseModified && $this->doUpdate()) {
//$this->out( "Database course changed " . $dbCourse->getName());
// Course has been modified
$this->out("UPDATE COURSE - " . $dbCourse->getName() . " - " . $dbCourse->getId());
$this->outputChangedFields($changedFields);
if ($this->doModify()) {
$em->persist($dbCourse);
$em->flush();
if ($udacityCourse['banner_image']) {
$this->uploadImageIfNecessary($udacityCourse['banner_image'], $dbCourse);
}
}
$courseChanged = true;
}
// Check if offering has been modified
$offering = $dbCourse->getNextOffering();
if ($offering->getUrl() != $course->getUrl() && $this->doUpdate()) {
$this->out("UPDATE COURSE - " . $dbCourse->getName() . " - " . $dbCourse->getId());
// Offering modified
$this->outputChangedFields(array(array('field' => 'offering Url', 'old' => $offering->getUrl(), 'new' => $course->getUrl())));
if ($this->doModify()) {
$offering->setUrl($course->getUrl());
$em->persist($offering);
$em->flush();
}
$courseChanged = true;
}
$course = $dbCourse;
}
}
if ($courseChanged) {
$coursesChanged[] = $course;
}
return $coursesChanged;
}
public function scrape()
{
$em = $this->getManager();
// Array of offerings created or updated
$offerings = array();
$this->out("Scraping " . $this->initiative->getName());
// Step 1: Getting a list of course URLs
$this->out("Getting a list of course pages");
$urls = $this->getListOfCoursePages();
$urlsCount = count($urls);
// Step 2: Go through the page and create/update offering
$this->out("Number of courses found: {$urlsCount}");
$this->out("Gathering details about each course");
$courseDetails = array();
foreach ($urls as $url) {
if (!$url) {
continue;
}
$courseDetail = array();
$this->domParser->load(file_get_contents(self::BASE_URL . $url));
// Ignore self paced
if (!$this->domParser->find('h2.offering_dates_date', 0)) {
continue;
}
// Get Name and shortName
$nameString = $this->domParser->find('h1.page-title', 0)->plaintext;
$openBracketPosition = strpos($nameString, '(');
$closeBracketPosition = strpos($nameString, ')');
$courseDetail['name'] = substr($nameString, 0, $openBracketPosition - 1);
$courseDetail['shortName'] = substr($nameString, $openBracketPosition + 1, $closeBracketPosition - $openBracketPosition - 1);
if ($courseDetail['name'] == 'Introduction to Nursing in Healthcar') {
$courseDetail['name'] = 'Introduction to Nursing in Healthcare';
$courseDetail['shortName'] = 'IntroNur';
}
// Get the video id from the url
// eg. www.youtube.com/embed/Bw8HkjGQb3U?wmode=opaque&rel=0&showinfo=0
$youtubeIdPosition = 31;
$video = 'http://' . $this->domParser->find('iframe.media-youtube-player', 0)->src;
$questionMarkPosition = strpos($video, '?');
$courseDetail['video'] = 'http://www.youtube.com/watch?v=' . substr($video, $youtubeIdPosition, $questionMarkPosition - $youtubeIdPosition);
$instructors = trim($this->domParser->find('div[id=subject-teacher-tagline]', 0)->plaintext);
// Remove the 'by'
$instructors = substr($instructors, 3);
$courseDetail['instructors'] = explode(' & ', $instructors);
$courseDetail['desc'] = $this->domParser->find('div.offering_body', 0)->plaintext;
$courseDetail['start_date'] = $this->domParser->find('h2.offering_dates_date', 0)->plaintext;
$courseDetail['end_date'] = $this->domParser->find('h2.offering_dates_date', 1)->plaintext;
$courseDetail['url'] = $url;
print_r($courseDetail);
$courseDetails[] = $courseDetail;
$this->domParser->clear();
}
$this->out(count($courseDetails) . ' course pages found');
// Default stream
$stream = $this->dbHelper->getStreamBySlug('business');
$this->out("Default stream is " . $stream->getName());
foreach ($courseDetails as $courseDetail) {
/**
* Taking a shortcut here. Check if a course is created or not. If it isn't create the
* course,offering, etc. Updates are ignored
* TODO: Not take a shortcut
*/
// Build a course object
$course = new Course();
$courseShortName = 'open2study_' . $courseDetail['shortName'];
$course->setShortName($courseShortName);
$course->setInitiative($this->initiative);
$course->setName($courseDetail['name']);
$course->setDescription($courseDetail['desc']);
$course->setStream($stream);
// Default to Business
$course->setVideoIntro($courseDetail['video']);
$course->setUrl(self::BASE_URL . $courseDetail['url']);
$dbCourse = $this->dbHelper->getCourseByShortName($courseShortName);
if (!$dbCourse) {
if ($this->doCreate()) {
// New course
$this->out("NEW COURSE - " . $course->getName());
if ($this->doModify()) {
foreach ($courseDetail['instructors'] as $instructor) {
$course->addInstructor($this->dbHelper->createInstructorIfNotExists($instructor));
}
$em->persist($course);
$em->flush();
}
}
} else {
$course = $dbCourse;
}
// Check if offering exists
$shortName = $this->getOfferingShortName($courseDetail);
$offering = $this->dbHelper->getOfferingByShortName($shortName);
if ($offering) {
continue;
}
// Check if create offering is oon
if (!$this->doCreate()) {
$offerings[] = $offering;
// Add it to the offerings table
continue;
}
$offering = new Offering();
$offering->setCourse($course);
$offering->setStartDate(\DateTime::createFromFormat("d/m/Y", $courseDetail['start_date']));
$offering->setEndDate(\DateTime::createFromFormat("d/m/Y", $courseDetail['end_date']));
$offering->setStatus(Offering::START_DATES_KNOWN);
$offering->setLength(4);
$offering->setShortName($shortName);
$offering->setUrl(self::BASE_URL . $courseDetail['url']);
$offering->setVideoIntro($courseDetail['video']);
$offering->setSearchDesc($courseDetail['desc']);
$offering->setCreated(new \DateTime());
if ($this->doModify()) {
try {
$em->persist($offering);
$em->flush();
$this->out("OFFERING {$courseDetail['name']} created");
} catch (\Exception $e) {
$this->out("OFFERING {$courseDetail['name']} creation FAILED");
}
}
$offerings[] = $offering;
}
return $offerings;
}
private function getOfferingEntity($canvasCourse, Course $course)
{
$offering = new Offering();
$offering->setCourse($course);
$offering->setUrl($course->getUrl());
$offering->setShortName('canvas_' . $canvasCourse['id']);
if ($canvasCourse['date'] == 'Self-paced') {
$startDate = new \DateTime();
$endDate = new \DateTime();
$endDate->add(new \DateInterval('P30D'));
$offering->setStatus(Offering::COURSE_OPEN);
$offering->setStartDate($startDate);
$offering->setEndDate($endDate);
} elseif (strpos($canvasCourse['date'], 'Start') === false) {
// Date is of the follow format: Jan 25 - Feb 29, 2016
$date = explode(',', $canvasCourse['date']);
$year = $date[1];
$day = explode('-', $date[0]);
$startDate = new \DateTime($day[0] . $year);
$endDate = new \DateTime($day[1] . ' ' . $year);
$offering->setStatus(Offering::START_DATES_KNOWN);
$offering->setStartDate($startDate);
$offering->setEndDate($endDate);
} else {
// Date contains Started ...
$date = substr($canvasCourse['date'], strpos($canvasCourse['date'], ' '));
$startDate = new \DateTime($date);
$endDate = new \DateTime('2018-12-31');
$offering->setStatus(Offering::COURSE_OPEN);
$offering->setStartDate($startDate);
$offering->setEndDate($endDate);
$this->out($offering->getDisplayDate());
}
return $offering;
}
private function getOfferingEntity($run, $course)
{
$offering = new Offering();
$offering->setShortName($run['uuid']);
$offering->setCourse($course);
$offering->setUrl($course->getUrl());
if ($run['start_date']) {
$startDate = new \DateTime($run['start_date']);
$endDate = new \DateTime($run['start_date']);
$days = $run['duration_in_weeks'] * 7;
$endDate->add(new \DateInterval("P{$days}D"));
$offering->setStartDate($startDate);
$offering->setEndDate($endDate);
$offering->setStatus(Offering::START_DATES_KNOWN);
} else {
$curYear = date('Y');
$startDate = new \DateTime("{$curYear}-12-30");
// Dec 30
$endDate = new \DateTime("{$curYear}-12-31");
// Dec 31
$offering->setStartDate($startDate);
$offering->setEndDate($endDate);
$offering->setStatus(Offering::START_YEAR_KNOWN);
}
return $offering;
}
