/**
* Creates a course if it does not exist
* @param $name Name of the course
* @param Initiative $initiative
* @param Institution $ins
* @return Course
*/
public function createCourseIfNotExists($name, Initiative $initiative, Institution $ins = null, Stream $stream)
{
// Check if course exists
$em = $this->scraper->getManager();
$courseRepository = $em->getRepository('ClassCentralSiteBundle:Course');
$course = $courseRepository->findOneBy(array('name' => $name, 'initiative' => $initiative->getId()));
// Course exists
if ($course) {
return $course;
}
$course = new Course();
$course->setName($name);
$course->setInitiative($initiative);
if ($ins) {
$course->addInstitution($ins);
}
$course->setStream($stream);
// Check if course is to be created
if ($this->scraper->doModify() && $this->scraper->doCreate()) {
$em->persist($course);
$em->flush();
$this->scraper->out("COURSE {$name} created for initiative " . $initiative->getName());
}
return $course;
}
/**
* Build a doctrine Course Entity out of a csv row
* @param $row
* @return Course
*/
public function getCourseEntity($row)
{
$course = new Course();
$course->setName($row[0]);
$course->setDescription($row[1]);
$course->setVideoIntro(str_replace('http', 'https', $row[4]));
$course->setUrl($row[5]);
$course->setShortName($this->getCourseId($row[5]));
$course->setInitiative($this->initiative);
// Set the language to arabic
$langMap = $this->dbHelper->getLanguageMap();
$course->setLanguage($langMap['Arabic']);
// Set the default stream as humanities
$defaultStream = $this->dbHelper->getStreamBySlug('humanities');
$course->setStream($defaultStream);
// Calculate the length of the course
$start = new \DateTime($row[2]);
$end = new \DateTime($row[3]);
$length = ceil($start->diff($end)->days / 7);
$course->setLength($length);
return $course;
}
public function migrate()
{
$this->output->writeln("Getting Started with migration version 1");
// Get all offerings
$em = $this->container->get('Doctrine')->getManager();
$offerings = $em->getRepository('ClassCentralSiteBundle:Offering')->findAll();
foreach ($offerings as $offering) {
$name = $offering->getName();
$initiative = null;
if ($offering->getInitiative()) {
$initiative = $offering->getInitiative();
}
$stream = $offering->getCourse()->getStream();
// Check if the course name and initiative exist
$initiative_id = $initiative ? $initiative->getID() : null;
$course = $em->getRepository('ClassCentralSiteBundle:Course')->findOneBy(array('name' => $name, 'initiative' => $initiative_id));
if (!$course) {
// Course does not exist. Create the course
//$this->output->writeln("NOT FOUND");
$course = new Course();
$course->setName($name);
$course->setInitiative($initiative);
$course->setStream($stream);
$em->persist($course);
$em->flush();
$this->output->writeln("Course '" . $course->getName() . "' created with " . $course->getId());
} else {
$this->output->writeln("Course '" . $course->getName() . " already exists with " . $course->getId());
}
// Update the course id in offering
$offering->setCourse($course);
$em->persist($offering);
$em->flush();
}
// Delete all the courses with course id 100
$em->createQuery("DELETE FROM ClassCentralSiteBundle:Course c WHERE c.initiative=100")->execute();
// Delete initative
$em->createQuery("DELETE FROM ClassCentralSiteBundle:Initiative i WHERE i.id = 100")->execute();
}
/**
* Given an array built from edX csv returns a course entity
* @param array $c
*/
private function getCourseEntity($c = array())
{
$defaultStream = $this->dbHelper->getStreamBySlug('cs');
$langMap = $this->dbHelper->getLanguageMap();
$defaultLanguage = $langMap['English'];
$course = new Course();
$course->setShortName($this->getShortName($c));
$course->setInitiative($this->initiative);
$course->setName($c['course-code'] . ': ' . $c['title']);
$course->setDescription($c['description']);
$course->setLongDescription(nl2br($c['description']));
$course->setLanguage($defaultLanguage);
$course->setStream($defaultStream);
// Default to Computer Science
$course->setVideoIntro($c['course-video-youtube']);
$course->setUrl($c['link']);
$course->setCertificate(false);
$course->setVerifiedCertificate($c['course-verified']);
// Calculate length
$length = null;
if (!empty($c['course-end'])) {
$start = new \DateTime($c['course-start']);
$end = new \DateTime($c['course-end']);
$length = ceil($start->diff($end)->days / 7);
}
$course->setLength($length);
return $course;
}
private function getOnDemandCourse($data = array())
{
$dbLanguageMap = $this->dbHelper->getLanguageMap();
$course = new Course();
$course->setShortName(substr('coursera_' . $data['elements'][0]['slug'], 0, 49));
$course->setInitiative($this->initiative);
$course->setName($data['elements'][0]['name']);
$course->setDescription($data['elements'][0]['description']);
$course->setLongDescription(nl2br($data['elements'][0]['description']));
$course->setStream($this->dbHelper->getStreamBySlug('cs'));
// Default to Computer Science
$course->setUrl('https://www.coursera.org/learn/' . $data['elements'][0]['slug']);
$lang = self::$languageMap[$data['elements']['0']['primaryLanguageCodes'][0]];
if (isset($dbLanguageMap[$lang])) {
$course->setLanguage($dbLanguageMap[$lang]);
} else {
$this->out("Language not found " . $data['elements']['0']['primaryLanguageCodes'][0]);
}
$course->setCertificate(false);
$course->setVerifiedCertificate($data['elements'][0]['isVerificationEnabled']);
// Add the university
foreach ($data['linked']['partners.v1'] as $university) {
$ins = new Institution();
$ins->setName($university['name']);
$ins->setIsUniversity(true);
$ins->setSlug($university['shortName']);
$course->addInstitution($this->dbHelper->createInstitutionIfNotExists($ins));
}
foreach ($data['linked']['instructors.v1'] as $courseraInstructor) {
if (!empty($courseraInstructor['fullName'])) {
$insName = $courseraInstructor['fullName'];
} else {
$insName = $courseraInstructor['firstName'] . ' ' . $courseraInstructor['lastName'];
}
$course->addInstructor($this->dbHelper->createInstructorIfNotExists($insName));
}
// Get Course Details like Syllabus and length
$courseDetails = json_decode(file_get_contents(sprintf(self::ONDEMAND_OPENCOURSE_API, $data['elements'][0]['slug'])), true);
if (!empty($courseDetails)) {
$syllabus = '';
foreach ($courseDetails['courseMaterial']['elements'] as $item) {
$syllabus .= "<b>{$item['name']}</b><br/>{$item['description']}<br/><br/>";
}
$course->setSyllabus($syllabus);
}
// Calculate the length of the course
$schedule = json_decode(file_get_contents(sprintf(self::ONDEMAND_COURSE_SCHEDULE, $data['elements'][0]['id'])), true);
if (!empty($schedule)) {
$length = 0;
foreach ($schedule['elements'][0]['defaultSchedule']['periods'] as $period) {
$length += $period['numberOfWeeks'];
}
if ($length > 0) {
$course->setLength($length);
}
}
return $course;
}
private function getCourseEntity($udacityCourse = array())
{
$defaultStream = $this->dbHelper->getStreamBySlug('cs');
$langMap = $this->dbHelper->getLanguageMap();
$defaultLanguage = $langMap['English'];
$course = new Course();
$course->setShortName(substr('udacity_' . $udacityCourse['slug'], 0, 50));
$course->setInitiative($this->initiative);
$course->setName($udacityCourse['title']);
$course->setDescription($udacityCourse['short_summary']);
$course->setLanguage($defaultLanguage);
$course->setStream($defaultStream);
// Default to Computer Science
$course->setCertificate(false);
$course->setUrl($udacityCourse['homepage']);
$course->setSyllabus(nl2br($udacityCourse['syllabus']));
$course->setWorkloadMin(6);
$course->setWorkloadMax(6);
// Calculate length
$length = null;
$expectedDuration = $udacityCourse['expected_duration'];
if ($udacityCourse['expected_duration_unit'] == 'months') {
$length = $expectedDuration * 4;
} elseif ($udacityCourse['expected_duration_unit'] == 'weeks') {
$length = $expectedDuration;
}
$course->setLength($length);
// Calculate Description
$course->setLongDescription(nl2br($udacityCourse['summary'] . '<br/><br/><b>Why Take This Course?</b><br/>' . $udacityCourse['expected_learning']));
// Intro Video
if (!empty($udacityCourse['teaser_video']['youtube_url'])) {
$course->setVideoIntro($udacityCourse['teaser_video']['youtube_url']);
}
return $course;
}
public function scrape()
{
$em = $this->getManager();
// Array of offerings created or updated
$offerings = array();
$this->out("Scraping " . $this->initiative->getName());
// Step 1: Getting a list of course URLs
$this->out("Getting a list of course pages");
$urls = $this->getListOfCoursePages();
$urlsCount = count($urls);
// Step 2: Go through the page and create/update offering
$this->out("Number of courses found: {$urlsCount}");
$this->out("Gathering details about each course");
$courseDetails = array();
foreach ($urls as $url) {
if (!$url) {
continue;
}
$courseDetail = array();
$this->domParser->load(file_get_contents(self::BASE_URL . $url));
// Ignore self paced
if (!$this->domParser->find('h2.offering_dates_date', 0)) {
continue;
}
// Get Name and shortName
$nameString = $this->domParser->find('h1.page-title', 0)->plaintext;
$openBracketPosition = strpos($nameString, '(');
$closeBracketPosition = strpos($nameString, ')');
$courseDetail['name'] = substr($nameString, 0, $openBracketPosition - 1);
$courseDetail['shortName'] = substr($nameString, $openBracketPosition + 1, $closeBracketPosition - $openBracketPosition - 1);
if ($courseDetail['name'] == 'Introduction to Nursing in Healthcar') {
$courseDetail['name'] = 'Introduction to Nursing in Healthcare';
$courseDetail['shortName'] = 'IntroNur';
}
// Get the video id from the url
// eg. www.youtube.com/embed/Bw8HkjGQb3U?wmode=opaque&rel=0&showinfo=0
$youtubeIdPosition = 31;
$video = 'http://' . $this->domParser->find('iframe.media-youtube-player', 0)->src;
$questionMarkPosition = strpos($video, '?');
$courseDetail['video'] = 'http://www.youtube.com/watch?v=' . substr($video, $youtubeIdPosition, $questionMarkPosition - $youtubeIdPosition);
$instructors = trim($this->domParser->find('div[id=subject-teacher-tagline]', 0)->plaintext);
// Remove the 'by'
$instructors = substr($instructors, 3);
$courseDetail['instructors'] = explode(' & ', $instructors);
$courseDetail['desc'] = $this->domParser->find('div.offering_body', 0)->plaintext;
$courseDetail['start_date'] = $this->domParser->find('h2.offering_dates_date', 0)->plaintext;
$courseDetail['end_date'] = $this->domParser->find('h2.offering_dates_date', 1)->plaintext;
$courseDetail['url'] = $url;
print_r($courseDetail);
$courseDetails[] = $courseDetail;
$this->domParser->clear();
}
$this->out(count($courseDetails) . ' course pages found');
// Default stream
$stream = $this->dbHelper->getStreamBySlug('business');
$this->out("Default stream is " . $stream->getName());
foreach ($courseDetails as $courseDetail) {
/**
* Taking a shortcut here. Check if a course is created or not. If it isn't create the
* course,offering, etc. Updates are ignored
* TODO: Not take a shortcut
*/
// Build a course object
$course = new Course();
$courseShortName = 'open2study_' . $courseDetail['shortName'];
$course->setShortName($courseShortName);
$course->setInitiative($this->initiative);
$course->setName($courseDetail['name']);
$course->setDescription($courseDetail['desc']);
$course->setStream($stream);
// Default to Business
$course->setVideoIntro($courseDetail['video']);
$course->setUrl(self::BASE_URL . $courseDetail['url']);
$dbCourse = $this->dbHelper->getCourseByShortName($courseShortName);
if (!$dbCourse) {
if ($this->doCreate()) {
// New course
$this->out("NEW COURSE - " . $course->getName());
if ($this->doModify()) {
foreach ($courseDetail['instructors'] as $instructor) {
$course->addInstructor($this->dbHelper->createInstructorIfNotExists($instructor));
}
$em->persist($course);
$em->flush();
}
}
} else {
$course = $dbCourse;
}
// Check if offering exists
$shortName = $this->getOfferingShortName($courseDetail);
$offering = $this->dbHelper->getOfferingByShortName($shortName);
if ($offering) {
continue;
}
// Check if create offering is oon
if (!$this->doCreate()) {
$offerings[] = $offering;
// Add it to the offerings table
continue;
}
$offering = new Offering();
$offering->setCourse($course);
$offering->setStartDate(\DateTime::createFromFormat("d/m/Y", $courseDetail['start_date']));
$offering->setEndDate(\DateTime::createFromFormat("d/m/Y", $courseDetail['end_date']));
$offering->setStatus(Offering::START_DATES_KNOWN);
$offering->setLength(4);
$offering->setShortName($shortName);
$offering->setUrl(self::BASE_URL . $courseDetail['url']);
$offering->setVideoIntro($courseDetail['video']);
$offering->setSearchDesc($courseDetail['desc']);
$offering->setCreated(new \DateTime());
if ($this->doModify()) {
try {
$em->persist($offering);
$em->flush();
$this->out("OFFERING {$courseDetail['name']} created");
} catch (\Exception $e) {
$this->out("OFFERING {$courseDetail['name']} creation FAILED");
}
}
$offerings[] = $offering;
}
return $offerings;
}
public function getCourse($canvasCourse)
{
$dbLanguageMap = $this->dbHelper->getLanguageMap();
$course = new Course();
$course->setName($canvasCourse['title']);
$course->setInitiative($this->initiative);
$course->setDescription($canvasCourse['teaser']);
$course->setUrl($canvasCourse['url']);
$course->setLanguage($dbLanguageMap['English']);
$course->setStream($this->dbHelper->getStreamBySlug('cs'));
// Default to Computer Science
$course->setShortName('canvas_' . $this->getSlug($canvasCourse['path']));
return $course;
}
private function getOnDemandCourse($data = array())
{
$dbLanguageMap = $this->dbHelper->getLanguageMap();
$course = new Course();
$course->setShortName(substr('coursera_' . $data['elements'][0]['slug'], 0, 49));
$course->setInitiative($this->initiative);
$course->setName($data['elements'][0]['name']);
$course->setDescription($data['elements'][0]['description']);
$course->setLongDescription(nl2br($data['elements'][0]['description']));
$course->setStream($this->dbHelper->getStreamBySlug('cs'));
// Default to Computer Science
$course->setUrl('https://www.coursera.org/learn/' . $data['elements'][0]['slug']);
$lang = self::$languageMap[$data['elements']['0']['primaryLanguageCodes'][0]];
if (isset($dbLanguageMap[$lang])) {
$course->setLanguage($dbLanguageMap[$lang]);
} else {
$this->out("Language not found " . $data['elements']['0']['primaryLanguageCodes'][0]);
}
$course->setCertificate(false);
$course->setVerifiedCertificate($data['elements'][0]['isVerificationEnabled']);
// Add the university
foreach ($data['linked']['partners.v1'] as $university) {
$ins = new Institution();
$ins->setName($university['name']);
$ins->setIsUniversity(true);
$ins->setSlug($university['shortName']);
$course->addInstitution($this->dbHelper->createInstitutionIfNotExists($ins));
}
foreach ($data['linked']['instructors.v1'] as $courseraInstructor) {
if (!empty($courseraInstructor['fullName'])) {
$insName = $courseraInstructor['fullName'];
} else {
$insName = $courseraInstructor['firstName'] . ' ' . $courseraInstructor['lastName'];
}
$course->addInstructor($this->dbHelper->createInstructorIfNotExists($insName));
}
return $course;
}
private function getCourseEntity($c = array())
{
$defaultStream = $this->dbHelper->getStreamBySlug('cs');
$langMap = $this->dbHelper->getLanguageMap();
$defaultLanguage = $langMap['English'];
$course = new Course();
$course->setShortName($c['uuid']);
$course->setInitiative($this->initiative);
$course->setName($c['name']);
$course->setDescription($c['introduction']);
$course->setLongDescription($c['description']);
$course->setLanguage($defaultLanguage);
$course->setStream($defaultStream);
// Default to Computer Science
$course->setUrl($c['url']);
$course->setCertificate($c['has_certificates']);
$course->setWorkloadMin($c['hours_per_week']);
$course->setWorkloadMax($c['hours_per_week']);
// Get the length
if ($c['runs']) {
$course->setLength($c['runs'][0]['duration_in_weeks']);
}
return $course;
}
