/** * Gets IMAP content * * @param string $imapHost * @param string $imapUser * @param string $imapPassword * @param \Swiftriver\Core\ObjectModel\Channel $channel * * @return $contentItems[] */ private function GetIMAPContent($imapHost, $imapUser, $imapPassword, $channel) { $imapResource = imap_open("{" . $imapHost . "}INBOX", $imapUser, $imapPassword); //Open up unseen messages $imapEmails = imap_search($imapResource, strtoupper($channel->subType)); $contentItems = array(); if ($imapEmails) { //Put newest emails on top rsort($imapEmails); foreach ($imapEmails as $Email) { //Loop through each email and return the content $email_overview = imap_fetch_overview($imapResource, $Email, 0); $email_message = imap_fetchbody($imapResource, $Email, 2); $source_name = $imapUser; $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name); $source->name = $source_name; $source->parent = $channel->id; $source->type = $channel->type; $source->subType = $channel->subType; $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source); $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $email_overview[0]->subject, array($email_message)); //the message $item->link = null; $item->date = $email_overview[0]->date; $contentItems[] = $item; } imap_close($imapResource); return $contentItems; } imap_close($imapResource); return null; }
/** * Gets IMAP content * * @param string $imapHost * @param string $imapUser * @param string $imapPassword * @param \Swiftriver\Core\ObjectModel\Channel $channel * * @return $contentItems[] */ private function GetIMAPContent($imapHost, $imapUser, $imapPassword, $channel) { $imapResource = imap_open("{" . $imapHost . "}INBOX", $imapUser, $imapPassword); //Open up unseen messages $search = $channel->lastSuccess == null ? "UNSEEN" : "UNSEEN SINCE " . \date("Y-m-d", $channel->lastSuccess); $imapEmails = imap_search($imapResource, $search); $contentItems = array(); if ($imapEmails) { //Put newest emails on top rsort($imapEmails); foreach ($imapEmails as $Email) { //Loop through each email and return the content $email_overview = imap_fetch_overview($imapResource, $Email, 0); if (strtotime(reset($email_overview)->date) < $channel->lastSuccess) { continue; } $email_header_info = imap_header($imapResource, $Email); $email_message = imap_fetchbody($imapResource, $Email, 1); $source_name = \reset($email_overview)->from; $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name); $source->name = $source_name; $source->parent = $channel->id; $source->type = $channel->type; $source->subType = $channel->subType; $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source); $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $email_overview[0]->subject, array($email_message)); //the message $item->link = null; $item->date = $email_header_info->udate; $contentItems[] = $item; } } imap_close($imapResource); return $contentItems; }
/** * Implementation of IParser::GetAndParse * @param \Swiftriver\Core\ObjectModel\Channel $channel * @param datetime $lassucess */ public function GetAndParse($channel) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [Method invoked]", \PEAR_LOG_DEBUG); $contentItems = array(); $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [START: Extracting required parameters]", \PEAR_LOG_DEBUG); //Extract the required variables $apikey = $channel->parameters["APIKEY"]; if (!isset($apikey) || $apikey == "") { $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [the parameter 'APIKEY' was not supplied. Returning null]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG); return null; } $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [END: Extracting required parameters]", \PEAR_LOG_DEBUG); try { $json = \file_get_contents("http://api.eventful.com/rest/demands/members/list?app_key=" . $apikey); $object = \json_decode($json); $source_name = "Eventful"; $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $channel->trusted); $source->name = $source_name; $source->link = "http://eventful.com"; $source->parent = $channel->id; $source->type = $channel->type; $source->subType = $channel->subType; foreach ($object->members as $event) { if (!isset($event->time_stamp) || !isset($event->longitude) || !isset($event->latitude) || !isset($event->location) || !isset($event->unique_id)) { $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [One event did not have the required parameters]", \PEAR_LOG_ERR); continue; } $contentdate = \strtotime($event->time_stamp); if (isset($channel->lastSuccess) && is_numeric($channel->lastSuccess) && isset($contentdate) && is_numeric($contentdate)) { if ($contentdate < $channel->lastSuccess) { $textContentDate = date("c", $contentdate); $textlastSuccess = date("c", $channel->lastSuccess); $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [Skipped feed item as date {$textContentDate} less than last sucessful run ({$textlastSuccess})]", \PEAR_LOG_DEBUG); continue; } } $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source); //Fill the Content Item $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, "Eventful event in " . $event->location, array("Eventful event in " . $event->location)); $item->link = "http://eventful.com"; $item->date = $contentdate; $item->gisData[] = new \Swiftriver\Core\ObjectModel\GisData($event->longitude, $event->latitude, $event->location); $contentItems[] = $item; } } catch (\Exception $e) { $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [{$e}]", \PEAR_LOG_ERR); } $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG); //return the content array return $contentItems; }
/** * Implementation of IPushParser::PushAndParse * @param $raw_content * @param $post_content * @param $get_content * @param $file_content * @return \Swiftriver\Core\ObjectModel\Content[] contentItems */ public function PushAndParse($raw_content = null, $post_content = null, $get_content = null, $file_content = null) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::Modules::SiSPS::PushParsers::CSVParser::PushAndParse [Method invoked]", \PEAR_LOG_INFO); $logger->log("Core::Modules::SiSPS::PushParsers::CSVParser::PushAndParse [START: Extracting required parameters]", \PEAR_LOG_INFO); $contentItems = array(); $settings = $this->GetSettings(); $file_name = "file_upload_" . $settings["file_upload_id"]; if (!isset($file_content[$file_name])) { $logger->log("Core::Modules::SiSPS::PushParsers::CSVParser::PushAndParse [No file uploaded with file_upload file name]", \PEAR_LOG_ERR); return $contentItems; } $logger->log("Core::Modules::SiSPS::PushParsers::CSVParser::PushAndParse [END: Extracting required parameters]", \PEAR_LOG_INFO); $file_handle = fopen($file_content[$file_name]["tmp_name"], "r"); if ($file_handle) { while (($data = fgetcsv($file_handle, 0, ",")) !== FALSE) { $title = $data[0]; $text = $data[1]; $link = $data[2]; $date = $data[3]; if (strrpos($date, "-")) { // Convert date to timestamp $date = explode("-", $date); $date = strptime($date[0] . "-" . $date[1] . "-" . $date[2], "%Y-%m-%d"); $date = mktime(0, 0, 0, $date['tm_month'], $date['tm_day'], $date['tm_year']); } $source_name = $this->ReturnType(); $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $settings["trusted"]); $source->parent = $this->ReturnType(); $source->name = $source_name; $source->link = $link; $source->type = $this->ReturnType(); $source->subType = $this->ReturnType(); //Create a new Content item $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source); //Fill the Content Item $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $title, array($text)); $item->link = $link; $item->date = $date; //Add the item to the Content array $contentItems[] = $item; } } else { $logger->log("Core::Modules::SiSPS::PushParsers::CSVParser::PushAndParse [Method finished]", \PEAR_LOG_INFO); return $contentItems; } fclose($file_handle); //return the content array return $contentItems; }
public function ParseJSONToSource($json) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::Workflows::SourceServices::SourceServicesBase::ParseJSONToSource [Method invoked]", \PEAR_LOG_DEBUG); $logger->log("Core::Workflows::SourceServices::SourceServicesBase::ParseJSONToSource [START: Creating new source]", \PEAR_LOG_DEBUG); try { //Try and get a source from the factory $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromJSON($json); } catch (\Exception $e) { //If exception, get the mesasge $message = $e->getMessage(); //and log it $logger->log("Core::Workflows::SourceServices::SourceServicesBase::ParseJSONToSource [{$message}]", \PEAR_LOG_ERR); $logger->log("Core::Workflows::SourceServices::SourceServicesBase::ParseJSONToSource [Method finished]", \PEAR_LOG_INFO); throw new \InvalidArgumentException("The JSON passed to this method did not contain data required to construct a source object: {$message}"); } $logger->log("Core::Workflows::SourceServices::SourceServicesBase::ParseJSONToSource [END: Creating new source]", \PEAR_LOG_DEBUG); $logger->log("Core::Workflows::SourceServices::SourceServicesBase::ParseJSONToSource [Method finished]", \PEAR_LOG_DEBUG); return $source; }
private function getGatewayContentItems($channel, $logger, $serverURL) { $contentItems = array(); $logger->log("Core::Modules::SiSPS::Parsers::SMSGatewayParser::getRemoteContentItems [Preparing URL]", \PEAR_LOG_DEBUG); $endServerURL = "/index.php/welcome/getsms/"; if (isset($channel->lastSucess)) { $endServerURL .= $channel->lastSucess; } else { $endServerURL .= "0"; } $logger->log("Core::Modules::SiSPS::Parsers::SMSGatewayParser::getRemoteContentItems [Connecting to server: " . $serverURL . $endServerURL . "]", \PEAR_LOG_DEBUG); $json_response = file_get_contents($serverURL . $endServerURL); $json_decoded = json_decode($json_response); $logger->log("Core::Modules::SiSPS::Parsers::SMSGatewayParser::getRemoteContentItems [Extracting message]", \PEAR_LOG_DEBUG); if (is_array($json_decoded->result)) { $num_messages = count($json_decoded->result); $json_decoded = $json_decoded->result; $logger->log("Core::Modules::SiSPS::Parsers::SMSGatewayParser::getRemoteContentItems [Processing {$num_messages} messages]", \PEAR_LOG_DEBUG); for ($message_index = 0; $message_index < $num_messages; $message_index++) { $source_name = ""; // Embed source name and source number $source_name = $json_decoded[$message_index]->message_from; $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $channel->trusted); $source->name = $source_name; $source->parent = $channel->id; $source->type = $channel->type; $source->subType = $channel->subType; //Create a new Content item $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source); $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText("", "", array($json_decoded[$message_index]->message_text, null)); //the message $item->link = null; $item->date = time(); $contentItems[] = $item; } return $contentItems; } else { return null; } }
/** * Given a set of parameters, this method should * fetch content from a channel and parse each * content into the Swiftriver object model : * Content Item. The $lastSuccess datetime is passed * to the function to ensure that content that has * already been parsed is not duplicated. * * @param \Swiftriver\Core\ObjectModel\Channel $channel * @return Swiftriver\Core\ObjectModel\Content[] contentItems */ public function GetAndParse($channel) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::Modules::SiSPS::Parsers::MeetupParser::GetAndParse [Method invoked]", \PEAR_LOG_DEBUG); $contentItems = array(); $apikey = $channel->parameters["APIKey"]; $urlname = $channel->parameters["urlname"]; $url = "http://api.meetup.com/ew/events?urlname={$urlname}&key={$apikey}&format=xml"; $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier("Meetup " . $urlname); $source->name = "Meetup {$urlname}"; $source->link = "http://meetup.com/{$urlname}"; $source->type = $channel->type; $source->subType = $channel->subType; $source->parent = $channel->id; $xml = \file_get_contents($url); $element = new \SimpleXmlElement($xml); foreach ($element->items->item as $item) { $rawdate = (string) $item->created; $date = (0 + $rawdate) / 1000; $lastSuccess = $channel->lastSuccess; if (isset($lastSuccess) && is_numeric($lastSuccess) && isset($date) && is_numeric($date)) { if ($date < $lastSuccess) { $textContentDate = date("c", $date); $textlastSuccess = date("c", $lastSuccess); $logger->log("Core::Modules::SiSPS::Parsers::MeetupParser::GetAndParse [Skipped feed item as date {$textContentDate} less than last sucessful run ({$textlastSuccess})]", \PEAR_LOG_DEBUG); continue; } } $content = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source); $content->date = $date; $content->gisData = array(new \Swiftriver\Core\ObjectModel\GisData((double) $item->lon, (double) $item->lat, "")); $content->link = (string) $item->meetup_url; $content->text = array(new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, "Meetup in " . (string) $item->city, array((string) $item->description))); $contentItems[] = $content; } $logger->log("Core::Modules::SiSPS::Parsers::MeetupParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG); //return the content array return $contentItems; }
/** * Implementation of IPushParser::PushAndParse * @param $raw_content * @param $post_content * @param $get_content * @param $file_content * @return \Swiftriver\Core\ObjectModel\Content[] contentItems */ public function PushAndParse($raw_content = null, $post_content = null, $get_content = null, $file_content = null) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::Modules::SiSPS::PushParsers::QuiverParser::PushAndParse [Method invoked]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::PushParsers::QuiverParser::PushAndParse [START: Extracting required parameters]", \PEAR_LOG_DEBUG); $settings = $this->GetSettings(); $source_name = $this->ReturnType(); $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $settings["trusted"]); $source->parent = $this->ReturnType(); $source->name = $source_name; $source->link = $get_content["u"]; $source->type = $this->ReturnType(); $source->subType = $this->ReturnType(); //Create a new Content item $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source); //Fill the Content Item $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $get_content["s"], array($get_content["s"])); $item->link = $get_content["u"]; $item->date = time(); //Add the item to the Content array $contentItems[] = $item; //return the content array return $contentItems; }
function GetSimplePieContentEntries($feedUrl, $logger, $channel) { //Include the Simple Pie Framework to get and parse feeds $config = \Swiftriver\Core\Setup::Configuration(); $simplePiePath = $config->ModulesDirectory . "/SimplePie/simplepie.inc"; include_once $simplePiePath; //Include the Simple Pie YouTube Framework $simpleTubePiePath = $config->ModulesDirectory . "/SimplePie/simpletube.inc"; include_once $simpleTubePiePath; $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [END: Including the SimplePie module]", \PEAR_LOG_DEBUG); //Construct a new SimplePie Parser $feed = new \SimplePie(); //Get the cache directory $cacheDirectory = $config->CachingDirectory; $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Setting the caching directory to {$cacheDirectory}]", \PEAR_LOG_DEBUG); //Set the caching directory $feed->set_cache_location($cacheDirectory); $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Setting the feed url to {$feedUrl}]", \PEAR_LOG_DEBUG); //Pass the feed URL to the SImplePie object $feed->set_feed_url($feedUrl); $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Initializing the feed]", \PEAR_LOG_DEBUG); //Run the SimplePie $feed->init(); //Strip HTML $feed->strip_htmltags(array('span', 'font', 'style', 'table', 'td', 'tr', 'div', 'p', 'br', 'a')); //Create the Content array $contentItems = array(); $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [START: Parsing feed items]", \PEAR_LOG_DEBUG); $feeditems = $feed->get_items(); if (!$feeditems || $feeditems == null || !is_array($feeditems) || count($feeditems) < 1) { $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [No feeditems recovered from the feed]", \PEAR_LOG_DEBUG); } $lastSuccess = $channel->lastSuccess; //Loop through the Feed Items foreach ($feeditems as $feedItem) { //Extract the date of the content $contentdate = strtotime($feedItem->get_date()); if (isset($lastSuccess) && is_numeric($lastSuccess) && isset($contentdate) && is_numeric($contentdate)) { if ($contentdate < $lastSuccess) { $textContentDate = date("c", $contentdate); $textlastSuccess = date("c", $lastSuccess); $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Skipped feed item as date {$textContentDate} less than last sucessful run ({$textlastSuccess})]", \PEAR_LOG_DEBUG); continue; } } $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Adding feed item]", \PEAR_LOG_DEBUG); //Get source data $source_name = $feedItem->get_author()->name; $source_name = $source_name == null || $source_name == "" ? "Google News Search -" . $this->searchPhrase : $source_name . " @ " . "Google News Search - " . $this->searchPhrase; $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $channel->trusted); $source->name = $source_name; $source->email = $feedItem->get_author()->email; $source->parent = $channel->id; $source->type = $channel->type; $source->subType = $channel->subType; //Extract all the relevant feedItem info $title = $feedItem->get_title(); $description = $feedItem->get_description(); $contentLink = $feedItem->get_permalink(); $date = $feedItem->get_date(); //Create a new Content item $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source); //Fill the Content Item $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $title, array($description)); $item->link = $contentLink; $item->date = strtotime($date); //Add the item to the Content array $contentItems[] = $item; } $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [END: Parsing feed items]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Method finished]", \PEAR_LOG_DEBUG); //return the content array return $contentItems; }
/** * Given an array of content is's, this function will * fetch the content objects from the data store. * * @param string[] $ids * @return \Swiftriver\Core\ObjectModel\Content[] */ public static function GetContent($ids, $orderby = null) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [Method invoked]", \PEAR_LOG_DEBUG); //if no $orderby is sent if (!$orderby || $orderby == null) { $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [No Order By clause set, setting to 'date desc']", \PEAR_LOG_DEBUG); //Set it to the default - date DESC $orderby = "date desc"; } //set up the return array $content = array(); //If the $ids array is blank or empty, return the empty array if (!isset($ids) || !is_array($ids) || count($ids) < 1) { $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [No IDs sent to method]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [Method finsiehd]", \PEAR_LOG_DEBUG); return $content; } $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [START: Building the RedBean Query]", \PEAR_LOG_DEBUG); //set up the array to hold the ids $queryIds = array(); //start to build the sql $query = "textId in ("; /*//for each content item, add to the query and the ids array for($i=0; $i<count($ids); $i++) { $query .= ":id$i,"; $queryIds[":id$i"] = $ids[$i]; }*/ $counter = 0; foreach ($ids as $id) { $query .= ":id{$counter},"; $queryIds[":id{$counter}"] = $id; $counter++; } //tidy up the query $query = rtrim($query, ",") . ") order by " . $orderby; $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [END: Building the RedBean Query]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [START: Running RedBean Query]", \PEAR_LOG_DEBUG); //Get the finder $finder = RedBeanController::Finder(); //Find the content $dbContent = $finder->where("content", $query, $queryIds); $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [FINISHED: Running RedBean Query]", \PEAR_LOG_DEBUG); //set up the return array $content = array(); //set up the red bean $rb = RedBeanController::RedBean(); $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [START: Constructing Content and Source items]", \PEAR_LOG_DEBUG); //loop through the db content foreach ($dbContent as $key => $dbItem) { //get the associated source $s = reset($rb->batch("source", RedBeanController::GetRelatedBeans($dbItem, "source"))); //Create the source from the db json $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromJSON($s->json); //get the json for the content $json = $dbItem->json; //create the content $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source, $json); //add it to the array $content[] = $item; } $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [END: Constructing Content and Source items]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [Method finished]", \PEAR_LOG_DEBUG); //return the content return $content; }
/** * Parses the simplepie item to a content item * @param \SimplePie_Item $tweet * @param \Swiftriver\Core\ObjectModel\Source * @return \Swiftriver\Core\ObjectModel\Content */ private function ParseTweetFromATOMItem($tweet, $channel) { //Extract all the relevant feedItem info $title = $tweet->get_title(); //$description = $tweet->get_description(); $contentLink = $tweet->get_permalink(); $date = $tweet->get_date(); //Create the source $author = $tweet->get_author(); $source_name = $author != null ? $author->get_name() : $channel->name; $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name); $source->name = $source_name; $source->email = $author != null ? $tweet->get_author()->get_email() : null; $source->link = $author != null ? $tweet->get_author()->get_link() : null; $source->parent = $channel->id; $source->type = $channel->type; $source->subType = $channel->subType; //Create a new Content item $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source); //Fill the Content Item $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $title, array()); $item->link = $contentLink; $item->date = strtotime($date); return $item; }
/** * Gets content items from the server * @param \Swiftriver\Core\ObjectModel\Channel $channel * @param string $serverURL * * @return array() */ private function getLocalContentItems($channel, $logger) { $contentItems = array(); $logger->log("Core::Modules::SiSPS::Parsers::FrontlineSMSParser::getLocalContentItems [Preparing to read from DB]", \PEAR_LOG_DEBUG); $lastSuccess = "0"; if (isset($channel->lastSucess)) { $lastSuccess = date('U', $channel->lastSucess) . "000"; } if ($channel->parameters["Database"] == "" && $channel->parameters[""] == "UserName") { $logger->log("Core::Modules::SiSPS::Parsers::FrontlineSMSParser::getLocalContentItems [Database and UserName parameters not set, exiting]", \PEAR_LOG_DEBUG); return null; } $my_connection = $this->localDBOpen("localhost", $channel->parameters["UserName"], $channel->parameters["Password"], $channel->parameters["Database"]); $my_messages = $this->localDBReturnMessages($my_connection, "message", $lastSuccess); $this->localDBClose($my_connection); if (is_array($my_messages)) { $num_messages = count($my_messages); $logger->log("Core::Modules::SiSPS::Parsers::FrontlineSMSParser::getContentItems [Processing {$num_messages} messages]", \PEAR_LOG_DEBUG); for ($message_index = 0; $message_index < $num_messages; $message_index++) { $source_name = ""; // Embed source name and source number $source_name = $my_messages[$message_index]['senderMsisdn']; $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $channel->trusted); $source->name = $source_name; $source->parent = $channel->id; $source->type = $channel->type; $source->subType = $channel->subType; //Create a new Content item $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source); $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText("", "", array($my_messages[$message_index]['textContent'], $my_messages[$message_index]['binaryMessageContent'])); //the message $item->link = null; $item->date = time(); $contentItems[] = $item; } return $contentItems; } else { return null; } }
/** * Implementation of IParser::GetAndParse * @param string[] $parameters * Required Parameter Values = * 'feedUrl' = The url to the RSS feed * @param datetime $lassucess */ public function GetAndParse($parameters, $lastsucess) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Method invoked]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [START: Extracting required parameters]", \PEAR_LOG_DEBUG); //Extract the required variables $feedUrl = $parameters["feedUrl"]; if (!isset($feedUrl) || $feedUrl == "") { $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [the parapeter 'feedUrl' was not supplued. Returning null]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG); return null; } $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [END: Extracting required parameters]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [START: Constructing source object]", \PEAR_LOG_DEBUG); //Create the source that will be used by all the content items Passing in the feed uri which can //be used to uniquly identify the source of the content $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromID($feedUrl); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [END: Constructing source object]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [START: Including the SimplePie module]", \PEAR_LOG_DEBUG); //Include the Simple Pie Framework to get and parse feeds $config = \Swiftriver\Core\Setup::Configuration(); include_once $config->ModulesDirectory . "/SimplePie/simplepie.inc"; $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [END: Including the SimplePie module]", \PEAR_LOG_DEBUG); //Construct a new SimplePie Parsaer $feed = new \SimplePie(); //Get the cach directory $cacheDirectory = $config->CachingDirectory; $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Setting the caching directory to {$cacheDirectory}]", \PEAR_LOG_DEBUG); //Set the caching directory $feed->set_cache_location($cacheDirectory); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Setting the feed url to {$feedUrl}]", \PEAR_LOG_DEBUG); //Pass the feed URL to the SImplePie object $feed->set_feed_url($feedUrl); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Initilising the feed]", \PEAR_LOG_DEBUG); //Run the SimplePie $feed->init(); //Create the Content array $contentItems = array(); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [START: Parsing feed items]", \PEAR_LOG_DEBUG); $feeditems = $feed->get_items(); if (!$feeditems || $feeditems == null || !is_array($feeditems) || count($feeditems) < 1) { $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [No feeditems recovered from the feed]", \PEAR_LOG_DEBUG); } //Loop throught the Feed Items foreach ($feeditems as $feedItem) { //Extract the date of the content $contentdate = strtotime($feedItem->get_date()); if (isset($lastsucess) && is_numeric($lastsucess) && isset($contentdate) && is_numeric($contentdate)) { if ($contentdate < $lastsucess) { $textContentDate = date("c", $contentdate); $textLastSucess = date("c", $lastsucess); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Skipped feed item as date {$textContentDate} less than last sucessful run ({$textLastSucess})]", \PEAR_LOG_DEBUG); continue; } } $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Adding feed item]", \PEAR_LOG_DEBUG); //Extract all the relevant feedItem info $title = $feedItem->get_title(); $description = $feedItem->get_description(); $contentLink = $feedItem->get_permalink(); $date = $feedItem->get_date(); //Create a new Content item $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source); //Fill the Content Item $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $title, array($description)); $item->link = $contentLink; $item->date = strtotime($date); //Add the item to the Content array $contentItems[] = $item; } $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [END: Parsing feed items]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG); //return the content array return $contentItems; }
/** * Implementation of IParser::GetAndParse * @param \Swiftriver\Core\ObjectModel\Channel $channel * @param datetime $lassucess */ public function GetAndParse($channel) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Method invoked]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [START: Extracting required parameters]", \PEAR_LOG_DEBUG); if ($channel->subType == "Tag Search" || $channel->subType == "Tag Search with Location") { $rawTags = $channel->parameters["tags"]; if (!isset($rawTags) || $rawTags == null) { $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [the parameter 'tags' was not supplied. Returning null]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG); return null; } $tags = \explode(" ", $rawTags); $url = $channel->subType == "Tag Search" ? "http://api.flickr.com/services/feeds/photos_public.gne?format=rss2&tags=" : "http://www.flickr.com/services/feeds/geo?format=rss2&tags="; foreach ($tags as $tag) { $url .= "{$tag},"; } $url = \rtrim($url, ","); } elseif ($channel->subType == "Follow a User") { $userid = $channel->parameters["userid"]; if (!isset($userid) || $userid == null) { $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [the parameter 'userid' was not supplied. Returning null]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG); return null; } $url = "http://api.flickr.com/services/feeds/photos_public.gne?id={$userid}"; } else { $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [The subType supplied was not recognised]", \PEAR_LOG_ERR); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG); return null; } $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [END: Extracting required parameters]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [START: Including the SimplePie module]", \PEAR_LOG_DEBUG); //Include the Simple Pie Framework to get and parse feeds $config = \Swiftriver\Core\Setup::Configuration(); $simplePiePath = $config->ModulesDirectory . "/SimplePie/simplepie.inc"; include_once $simplePiePath; //Include the Simple Pie YouTube Framework $simpleTubePiePath = $config->ModulesDirectory . "/SimplePie/simpletube.inc"; include_once $simpleTubePiePath; $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [END: Including the SimplePie module]", \PEAR_LOG_DEBUG); //Construct a new SimplePie Parser $feed = new \SimplePie(); //Get the cache directory $cacheDirectory = $config->CachingDirectory; $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Setting the caching directory to {$cacheDirectory}]", \PEAR_LOG_DEBUG); //Set the caching directory $feed->set_cache_location($cacheDirectory); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Setting the feed url to {$url}]", \PEAR_LOG_DEBUG); //Pass the feed URL to the SImplePie object $feed->set_feed_url($url); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Initializing the feed]", \PEAR_LOG_DEBUG); //Run the SimplePie $feed->init(); //Strip HTML $feed->strip_htmltags(array('span', 'font', 'style', 'p')); //Create the Content array $contentItems = array(); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [START: Parsing feed items]", \PEAR_LOG_DEBUG); $feeditems = $feed->get_items(); if (!$feeditems || $feeditems == null || !is_array($feeditems) || count($feeditems) < 1) { $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [No feeditems recovered from the feed]", \PEAR_LOG_DEBUG); } $lastSuccess = $channel->lastSuccess; //Loop through the Feed Items foreach ($feeditems as $feedItem) { //Extract the date of the content $contentdate = strtotime($feedItem->get_date()); if (isset($lastSuccess) && is_numeric($lastSuccess) && isset($contentdate) && is_numeric($contentdate)) { if ($contentdate < $lastSuccess) { $textContentDate = date("c", $contentdate); $textlastSuccess = date("c", $lastSuccess); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Skipped feed item as date {$textContentDate} less than last sucessful run ({$textlastSuccess})]", \PEAR_LOG_DEBUG); continue; } } $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Adding feed item]", \PEAR_LOG_DEBUG); //Get source data $source_name = $feedItem->get_author()->name; if (!isset($source_name) || $source_name == null || $source_name == "") { $source_name = "unknown"; } $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $channel->trusted); $source->name = $source_name; $source->email = $feedItem->get_author()->email; $source->parent = $channel->id; $source->type = $channel->type; $source->subType = $channel->subType; //Extract all the relevant feedItem info $title = $feedItem->get_title(); $description = $feedItem->get_description(); $contentLink = $feedItem->get_permalink(); $date = $feedItem->get_date(); $rawLocation = $feedItem->get_item_tags("http://www.georss.org/georss", "point"); $long = 0; $lat = 0; $name = ""; if (is_array($rawLocation)) { $lat_lon_array = split(" ", $rawLocation[0]["data"]); $long = $lat_lon_array[1]; $lat = $lat_lon_array[0]; $location = new \Swiftriver\Core\ObjectModel\GisData($long, $lat, $name); } //Create a new Content item $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source); if (isset($location)) { $item->gisData = array($location); } //Fill the Content Item $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $title, array($description)); $item->link = $contentLink; $item->date = strtotime($date); //Add the item to the Content array $contentItems[] = $item; } $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [END: Parsing feed items]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG); //return the content array return $contentItems; }
/** * Lists all the current Source in the core * @return \Swiftriver\Core\ObjectModel\Source[] */ public static function ListAllSources() { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [Method initiated]", \PEAR_LOG_DEBUG); $sources = array(); $selectAllSourcesSql = "CALL SC_SelectAllSources ()"; try { $db = self::PDOConnection(); $selectAllSourcesStatment = $db->prepare($selectAllSourcesSql); $result = $selectAllSourcesStatment->execute(); if ($result === false) { $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [An Exception was thrown by the PDO framwork]", \PEAR_LOG_ERR); $errorInfo = $selectAllSourcesStatment->errorInfo(); $errorMessage = $errorInfo[2]; $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [{$errorMessage}]", \PEAR_LOG_ERR); } $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [END: Executing PDO statement]", \PEAR_LOG_DEBUG); foreach ($selectAllSourcesStatment->fetchAll() as $row) { $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [START: Constructing source obejct]", \PEAR_LOG_DEBUG); $json = $row["json"]; $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromJSON($json); $sources[] = $source; $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [END: Constructing source obejct]", \PEAR_LOG_DEBUG); } $db = null; } catch (\PDOException $e) { $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [An Exception was thrown:]", \PEAR_LOG_ERR); $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [{$e}]", \PEAR_LOG_ERR); } $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [Method finished]", \PEAR_LOG_DEBUG); return $sources; }
/** * Parses the simplepie item to a content item * @param \SimplePie_Item $tweet * @param \Swiftriver\Core\ObjectModel\Channel $channel */ private function ParseTweetFromATOMItem($tweet, $channel) { //Extract all the relevant feedItem info $title = $tweet->get_title(); //$description = $tweet->get_description(); $contentLink = $tweet->get_permalink(); $date = $tweet->get_date(); //Create the source $author = $tweet->get_author(); $source_name = $author != null ? $author->get_name() : $channel->name; $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $channel->trusted); $source->name = $source_name; $source->email = $author != null ? $tweet->get_author()->get_email() : null; $source->link = $author != null ? $tweet->get_author()->get_link() : null; $source->parent = $channel->id; $source->type = $channel->type; $source->subType = $channel->subType; //Add location data //Long and lat $location = $tweet->get_item_tags("http://www.georss.org/georss", "point"); $long = 0; $lat = 0; $name = ""; if (is_array($location)) { $lat_lon_array = split(" ", $location[0]["data"]); $long = $lat_lon_array[1]; $lat = $lat_lon_array[0]; //Name $location_name = $tweet->get_item_tags("http://api.twitter.com", "place"); if (is_array($location_name)) { if (isset($location_name[0]["child"]["http://api.twitter.com"]["full_name"][0]["data"])) { $name = $location_name[0]["child"]["http://api.twitter.com"]["full_name"][0]["data"]; } } $source->gisData = array(new \Swiftriver\Core\ObjectModel\GisData($long, $lat, $name)); } //Create a new Content item $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source); //Fill the Content Item $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $title, array()); $item->link = $contentLink; $item->date = strtotime($date); //Sanitize the tweet text into a DIF collection $sanitizedTweetDiffCollection = $this->ParseTweetToSanitizedTweetDiffCollection($tweet); //Add the dif collection to the item $item->difs = array($sanitizedTweetDiffCollection); //return the item return $item; }
/** * Processes a queue file and does something with it (example only) * @param string $queueFile The queue file */ protected function processQueueFile($queueFile) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::Modules::TwitterStreamingSearchClient start processing " . $queueFile, \PEAR_LOG_DEBUG); // Open file $fp = fopen($queueFile, 'r'); // Check if something has gone wrong, or perhaps the file is just locked by another process if (!is_resource($fp)) { $logger->log("Core::Modules::TwitterStreamingSearchClient file: " . $queueFile . " already open, skipping", \PEAR_LOG_DEBUG); return FALSE; } // Lock file flock($fp, LOCK_EX); $content = array(); $logger->log("Core::Modules::TwitterStreamingSearchClient: START Looping through content", \PEAR_LOG_DEBUG); while ($rawStatus = fgets($fp, 4096)) { try { $status = \json_decode($rawStatus); $source_name = $status->user->screen_name; if ($source_name == null || $source_name == "") { $source_name = "UNKNOWN"; } $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, false); $source->name = $source_name; $source->link = "http://twitter.com/" . $source_name; $source->parent = "TWITTERSTREAM"; $source->type = "Twitter Stream"; $source->subType = "Filter"; $source->applicationIds["twitter"] = $status->user->id; $source->applicationProfileImages["twitter"] = $status->user->profile_image_url; //Create a new Content item $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source); //Fill the Content Item $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $status->text, array($status->text)); $item->link = "http://twitter.com/" . $source_name . "/statuses/" . $status->id_str; $item->date = strtotime($status->created_at); /* GEO is not yet supported on this streamin feed if($tweet->geo != null && $tweet->geo->type == "Point" && \is_array($tweet->geo->coordinates)) $item->gisData[] = new \Swiftriver\Core\ObjectModel\GisData ( $tweet->geo->coordinates[1], $tweet->geo->coordinates[0], ""); */ //Sanitize the tweet text into a DIF collection $sanitizedTweetDiffCollection = $this->ParseTweetToSanitizedTweetDiffCollection($item); //Add the dif collection to the item $item->difs = array($sanitizedTweetDiffCollection); $content[] = $item; } catch (\Exception $e) { $logger->log("Core::Modules::TwitterStreamingSearchClient: {$e}", \PEAR_LOG_ERR); } } $logger->log("Core::Modules::TwitterStreamingSearchClient: START Looping through content", \PEAR_LOG_DEBUG); $workflow = new \Swiftriver\Core\Workflows\ContentServices\ProcessContent(); //Here we are running the workflow without pre processing. $workflow->RunWorkflow($content, false); // Release lock and close flock($fp, LOCK_UN); fclose($fp); // All done with this file $logger->log("Core::Modules::TwitterStreamingSearchClient finshed processing " . $queueFile, \PEAR_LOG_DEBUG); unlink($queueFile); }