Esempio n. 1
0
 /**
  * Gets IMAP content
  *
  * @param string $imapHost
  * @param string $imapUser
  * @param string $imapPassword
  * @param \Swiftriver\Core\ObjectModel\Channel $channel
  *
  * @return $contentItems[]
  */
 private function GetIMAPContent($imapHost, $imapUser, $imapPassword, $channel)
 {
     $imapResource = imap_open("{" . $imapHost . "}INBOX", $imapUser, $imapPassword);
     //Open up unseen messages
     $imapEmails = imap_search($imapResource, strtoupper($channel->subType));
     $contentItems = array();
     if ($imapEmails) {
         //Put newest emails on top
         rsort($imapEmails);
         foreach ($imapEmails as $Email) {
             //Loop through each email and return the content
             $email_overview = imap_fetch_overview($imapResource, $Email, 0);
             $email_message = imap_fetchbody($imapResource, $Email, 2);
             $source_name = $imapUser;
             $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name);
             $source->name = $source_name;
             $source->parent = $channel->id;
             $source->type = $channel->type;
             $source->subType = $channel->subType;
             $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source);
             $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $email_overview[0]->subject, array($email_message));
             //the message
             $item->link = null;
             $item->date = $email_overview[0]->date;
             $contentItems[] = $item;
         }
         imap_close($imapResource);
         return $contentItems;
     }
     imap_close($imapResource);
     return null;
 }
Esempio n. 2
0
 /**
  * Gets IMAP content
  *
  * @param string $imapHost
  * @param string $imapUser
  * @param string $imapPassword
  * @param \Swiftriver\Core\ObjectModel\Channel $channel
  *
  * @return $contentItems[]
  */
 private function GetIMAPContent($imapHost, $imapUser, $imapPassword, $channel)
 {
     $imapResource = imap_open("{" . $imapHost . "}INBOX", $imapUser, $imapPassword);
     //Open up unseen messages
     $search = $channel->lastSuccess == null ? "UNSEEN" : "UNSEEN SINCE " . \date("Y-m-d", $channel->lastSuccess);
     $imapEmails = imap_search($imapResource, $search);
     $contentItems = array();
     if ($imapEmails) {
         //Put newest emails on top
         rsort($imapEmails);
         foreach ($imapEmails as $Email) {
             //Loop through each email and return the content
             $email_overview = imap_fetch_overview($imapResource, $Email, 0);
             if (strtotime(reset($email_overview)->date) < $channel->lastSuccess) {
                 continue;
             }
             $email_header_info = imap_header($imapResource, $Email);
             $email_message = imap_fetchbody($imapResource, $Email, 1);
             $source_name = \reset($email_overview)->from;
             $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name);
             $source->name = $source_name;
             $source->parent = $channel->id;
             $source->type = $channel->type;
             $source->subType = $channel->subType;
             $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source);
             $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $email_overview[0]->subject, array($email_message));
             //the message
             $item->link = null;
             $item->date = $email_header_info->udate;
             $contentItems[] = $item;
         }
     }
     imap_close($imapResource);
     return $contentItems;
 }
Esempio n. 3
0
 /**
  * Implementation of IParser::GetAndParse
  * @param \Swiftriver\Core\ObjectModel\Channel $channel
  * @param datetime $lassucess
  */
 public function GetAndParse($channel)
 {
     $logger = \Swiftriver\Core\Setup::GetLogger();
     $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [Method invoked]", \PEAR_LOG_DEBUG);
     $contentItems = array();
     $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [START: Extracting required parameters]", \PEAR_LOG_DEBUG);
     //Extract the required variables
     $apikey = $channel->parameters["APIKEY"];
     if (!isset($apikey) || $apikey == "") {
         $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [the parameter 'APIKEY' was not supplied. Returning null]", \PEAR_LOG_DEBUG);
         $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG);
         return null;
     }
     $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [END: Extracting required parameters]", \PEAR_LOG_DEBUG);
     try {
         $json = \file_get_contents("http://api.eventful.com/rest/demands/members/list?app_key=" . $apikey);
         $object = \json_decode($json);
         $source_name = "Eventful";
         $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $channel->trusted);
         $source->name = $source_name;
         $source->link = "http://eventful.com";
         $source->parent = $channel->id;
         $source->type = $channel->type;
         $source->subType = $channel->subType;
         foreach ($object->members as $event) {
             if (!isset($event->time_stamp) || !isset($event->longitude) || !isset($event->latitude) || !isset($event->location) || !isset($event->unique_id)) {
                 $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [One event did not have the required parameters]", \PEAR_LOG_ERR);
                 continue;
             }
             $contentdate = \strtotime($event->time_stamp);
             if (isset($channel->lastSuccess) && is_numeric($channel->lastSuccess) && isset($contentdate) && is_numeric($contentdate)) {
                 if ($contentdate < $channel->lastSuccess) {
                     $textContentDate = date("c", $contentdate);
                     $textlastSuccess = date("c", $channel->lastSuccess);
                     $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [Skipped feed item as date {$textContentDate} less than last sucessful run ({$textlastSuccess})]", \PEAR_LOG_DEBUG);
                     continue;
                 }
             }
             $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source);
             //Fill the Content Item
             $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, "Eventful event in " . $event->location, array("Eventful event in " . $event->location));
             $item->link = "http://eventful.com";
             $item->date = $contentdate;
             $item->gisData[] = new \Swiftriver\Core\ObjectModel\GisData($event->longitude, $event->latitude, $event->location);
             $contentItems[] = $item;
         }
     } catch (\Exception $e) {
         $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [{$e}]", \PEAR_LOG_ERR);
     }
     $logger->log("Core::Modules::SiSPS::Parsers::EventfulParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG);
     //return the content array
     return $contentItems;
 }
Esempio n. 4
0
 /**
  * Implementation of IPushParser::PushAndParse
  * @param $raw_content
  * @param $post_content
  * @param $get_content
  * @param $file_content
  * @return \Swiftriver\Core\ObjectModel\Content[] contentItems
  */
 public function PushAndParse($raw_content = null, $post_content = null, $get_content = null, $file_content = null)
 {
     $logger = \Swiftriver\Core\Setup::GetLogger();
     $logger->log("Core::Modules::SiSPS::PushParsers::CSVParser::PushAndParse [Method invoked]", \PEAR_LOG_INFO);
     $logger->log("Core::Modules::SiSPS::PushParsers::CSVParser::PushAndParse [START: Extracting required parameters]", \PEAR_LOG_INFO);
     $contentItems = array();
     $settings = $this->GetSettings();
     $file_name = "file_upload_" . $settings["file_upload_id"];
     if (!isset($file_content[$file_name])) {
         $logger->log("Core::Modules::SiSPS::PushParsers::CSVParser::PushAndParse [No file uploaded with file_upload file name]", \PEAR_LOG_ERR);
         return $contentItems;
     }
     $logger->log("Core::Modules::SiSPS::PushParsers::CSVParser::PushAndParse [END: Extracting required parameters]", \PEAR_LOG_INFO);
     $file_handle = fopen($file_content[$file_name]["tmp_name"], "r");
     if ($file_handle) {
         while (($data = fgetcsv($file_handle, 0, ",")) !== FALSE) {
             $title = $data[0];
             $text = $data[1];
             $link = $data[2];
             $date = $data[3];
             if (strrpos($date, "-")) {
                 // Convert date to timestamp
                 $date = explode("-", $date);
                 $date = strptime($date[0] . "-" . $date[1] . "-" . $date[2], "%Y-%m-%d");
                 $date = mktime(0, 0, 0, $date['tm_month'], $date['tm_day'], $date['tm_year']);
             }
             $source_name = $this->ReturnType();
             $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $settings["trusted"]);
             $source->parent = $this->ReturnType();
             $source->name = $source_name;
             $source->link = $link;
             $source->type = $this->ReturnType();
             $source->subType = $this->ReturnType();
             //Create a new Content item
             $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source);
             //Fill the Content Item
             $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $title, array($text));
             $item->link = $link;
             $item->date = $date;
             //Add the item to the Content array
             $contentItems[] = $item;
         }
     } else {
         $logger->log("Core::Modules::SiSPS::PushParsers::CSVParser::PushAndParse [Method finished]", \PEAR_LOG_INFO);
         return $contentItems;
     }
     fclose($file_handle);
     //return the content array
     return $contentItems;
 }
 public function ParseJSONToSource($json)
 {
     $logger = \Swiftriver\Core\Setup::GetLogger();
     $logger->log("Core::Workflows::SourceServices::SourceServicesBase::ParseJSONToSource [Method invoked]", \PEAR_LOG_DEBUG);
     $logger->log("Core::Workflows::SourceServices::SourceServicesBase::ParseJSONToSource [START: Creating new source]", \PEAR_LOG_DEBUG);
     try {
         //Try and get a source from the factory
         $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromJSON($json);
     } catch (\Exception $e) {
         //If exception, get the mesasge
         $message = $e->getMessage();
         //and log it
         $logger->log("Core::Workflows::SourceServices::SourceServicesBase::ParseJSONToSource [{$message}]", \PEAR_LOG_ERR);
         $logger->log("Core::Workflows::SourceServices::SourceServicesBase::ParseJSONToSource [Method finished]", \PEAR_LOG_INFO);
         throw new \InvalidArgumentException("The JSON passed to this method did not contain data required to construct a source object: {$message}");
     }
     $logger->log("Core::Workflows::SourceServices::SourceServicesBase::ParseJSONToSource [END: Creating new source]", \PEAR_LOG_DEBUG);
     $logger->log("Core::Workflows::SourceServices::SourceServicesBase::ParseJSONToSource [Method finished]", \PEAR_LOG_DEBUG);
     return $source;
 }
 private function getGatewayContentItems($channel, $logger, $serverURL)
 {
     $contentItems = array();
     $logger->log("Core::Modules::SiSPS::Parsers::SMSGatewayParser::getRemoteContentItems [Preparing URL]", \PEAR_LOG_DEBUG);
     $endServerURL = "/index.php/welcome/getsms/";
     if (isset($channel->lastSucess)) {
         $endServerURL .= $channel->lastSucess;
     } else {
         $endServerURL .= "0";
     }
     $logger->log("Core::Modules::SiSPS::Parsers::SMSGatewayParser::getRemoteContentItems [Connecting to server: " . $serverURL . $endServerURL . "]", \PEAR_LOG_DEBUG);
     $json_response = file_get_contents($serverURL . $endServerURL);
     $json_decoded = json_decode($json_response);
     $logger->log("Core::Modules::SiSPS::Parsers::SMSGatewayParser::getRemoteContentItems [Extracting message]", \PEAR_LOG_DEBUG);
     if (is_array($json_decoded->result)) {
         $num_messages = count($json_decoded->result);
         $json_decoded = $json_decoded->result;
         $logger->log("Core::Modules::SiSPS::Parsers::SMSGatewayParser::getRemoteContentItems [Processing {$num_messages} messages]", \PEAR_LOG_DEBUG);
         for ($message_index = 0; $message_index < $num_messages; $message_index++) {
             $source_name = "";
             // Embed source name and source number
             $source_name = $json_decoded[$message_index]->message_from;
             $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $channel->trusted);
             $source->name = $source_name;
             $source->parent = $channel->id;
             $source->type = $channel->type;
             $source->subType = $channel->subType;
             //Create a new Content item
             $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source);
             $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText("", "", array($json_decoded[$message_index]->message_text, null));
             //the message
             $item->link = null;
             $item->date = time();
             $contentItems[] = $item;
         }
         return $contentItems;
     } else {
         return null;
     }
 }
Esempio n. 7
0
 /**
  * Given a set of parameters, this method should
  * fetch content from a channel and parse each
  * content into the Swiftriver object model :
  * Content Item. The $lastSuccess datetime is passed
  * to the function to ensure that content that has
  * already been parsed is not duplicated.
  *
  * @param \Swiftriver\Core\ObjectModel\Channel $channel
  * @return Swiftriver\Core\ObjectModel\Content[] contentItems
  */
 public function GetAndParse($channel)
 {
     $logger = \Swiftriver\Core\Setup::GetLogger();
     $logger->log("Core::Modules::SiSPS::Parsers::MeetupParser::GetAndParse [Method invoked]", \PEAR_LOG_DEBUG);
     $contentItems = array();
     $apikey = $channel->parameters["APIKey"];
     $urlname = $channel->parameters["urlname"];
     $url = "http://api.meetup.com/ew/events?urlname={$urlname}&key={$apikey}&format=xml";
     $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier("Meetup " . $urlname);
     $source->name = "Meetup {$urlname}";
     $source->link = "http://meetup.com/{$urlname}";
     $source->type = $channel->type;
     $source->subType = $channel->subType;
     $source->parent = $channel->id;
     $xml = \file_get_contents($url);
     $element = new \SimpleXmlElement($xml);
     foreach ($element->items->item as $item) {
         $rawdate = (string) $item->created;
         $date = (0 + $rawdate) / 1000;
         $lastSuccess = $channel->lastSuccess;
         if (isset($lastSuccess) && is_numeric($lastSuccess) && isset($date) && is_numeric($date)) {
             if ($date < $lastSuccess) {
                 $textContentDate = date("c", $date);
                 $textlastSuccess = date("c", $lastSuccess);
                 $logger->log("Core::Modules::SiSPS::Parsers::MeetupParser::GetAndParse [Skipped feed item as date {$textContentDate} less than last sucessful run ({$textlastSuccess})]", \PEAR_LOG_DEBUG);
                 continue;
             }
         }
         $content = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source);
         $content->date = $date;
         $content->gisData = array(new \Swiftriver\Core\ObjectModel\GisData((double) $item->lon, (double) $item->lat, ""));
         $content->link = (string) $item->meetup_url;
         $content->text = array(new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, "Meetup in " . (string) $item->city, array((string) $item->description)));
         $contentItems[] = $content;
     }
     $logger->log("Core::Modules::SiSPS::Parsers::MeetupParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG);
     //return the content array
     return $contentItems;
 }
 /**
  * Implementation of IPushParser::PushAndParse
  * @param $raw_content
  * @param $post_content
  * @param $get_content
  * @param $file_content
  * @return \Swiftriver\Core\ObjectModel\Content[] contentItems
  */
 public function PushAndParse($raw_content = null, $post_content = null, $get_content = null, $file_content = null)
 {
     $logger = \Swiftriver\Core\Setup::GetLogger();
     $logger->log("Core::Modules::SiSPS::PushParsers::QuiverParser::PushAndParse [Method invoked]", \PEAR_LOG_DEBUG);
     $logger->log("Core::Modules::SiSPS::PushParsers::QuiverParser::PushAndParse [START: Extracting required parameters]", \PEAR_LOG_DEBUG);
     $settings = $this->GetSettings();
     $source_name = $this->ReturnType();
     $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $settings["trusted"]);
     $source->parent = $this->ReturnType();
     $source->name = $source_name;
     $source->link = $get_content["u"];
     $source->type = $this->ReturnType();
     $source->subType = $this->ReturnType();
     //Create a new Content item
     $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source);
     //Fill the Content Item
     $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $get_content["s"], array($get_content["s"]));
     $item->link = $get_content["u"];
     $item->date = time();
     //Add the item to the Content array
     $contentItems[] = $item;
     //return the content array
     return $contentItems;
 }
 function GetSimplePieContentEntries($feedUrl, $logger, $channel)
 {
     //Include the Simple Pie Framework to get and parse feeds
     $config = \Swiftriver\Core\Setup::Configuration();
     $simplePiePath = $config->ModulesDirectory . "/SimplePie/simplepie.inc";
     include_once $simplePiePath;
     //Include the Simple Pie YouTube Framework
     $simpleTubePiePath = $config->ModulesDirectory . "/SimplePie/simpletube.inc";
     include_once $simpleTubePiePath;
     $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [END: Including the SimplePie module]", \PEAR_LOG_DEBUG);
     //Construct a new SimplePie Parser
     $feed = new \SimplePie();
     //Get the cache directory
     $cacheDirectory = $config->CachingDirectory;
     $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Setting the caching directory to {$cacheDirectory}]", \PEAR_LOG_DEBUG);
     //Set the caching directory
     $feed->set_cache_location($cacheDirectory);
     $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Setting the feed url to {$feedUrl}]", \PEAR_LOG_DEBUG);
     //Pass the feed URL to the SImplePie object
     $feed->set_feed_url($feedUrl);
     $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Initializing the feed]", \PEAR_LOG_DEBUG);
     //Run the SimplePie
     $feed->init();
     //Strip HTML
     $feed->strip_htmltags(array('span', 'font', 'style', 'table', 'td', 'tr', 'div', 'p', 'br', 'a'));
     //Create the Content array
     $contentItems = array();
     $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [START: Parsing feed items]", \PEAR_LOG_DEBUG);
     $feeditems = $feed->get_items();
     if (!$feeditems || $feeditems == null || !is_array($feeditems) || count($feeditems) < 1) {
         $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [No feeditems recovered from the feed]", \PEAR_LOG_DEBUG);
     }
     $lastSuccess = $channel->lastSuccess;
     //Loop through the Feed Items
     foreach ($feeditems as $feedItem) {
         //Extract the date of the content
         $contentdate = strtotime($feedItem->get_date());
         if (isset($lastSuccess) && is_numeric($lastSuccess) && isset($contentdate) && is_numeric($contentdate)) {
             if ($contentdate < $lastSuccess) {
                 $textContentDate = date("c", $contentdate);
                 $textlastSuccess = date("c", $lastSuccess);
                 $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Skipped feed item as date {$textContentDate} less than last sucessful run ({$textlastSuccess})]", \PEAR_LOG_DEBUG);
                 continue;
             }
         }
         $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Adding feed item]", \PEAR_LOG_DEBUG);
         //Get source data
         $source_name = $feedItem->get_author()->name;
         $source_name = $source_name == null || $source_name == "" ? "Google News Search -" . $this->searchPhrase : $source_name . " @ " . "Google News Search - " . $this->searchPhrase;
         $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $channel->trusted);
         $source->name = $source_name;
         $source->email = $feedItem->get_author()->email;
         $source->parent = $channel->id;
         $source->type = $channel->type;
         $source->subType = $channel->subType;
         //Extract all the relevant feedItem info
         $title = $feedItem->get_title();
         $description = $feedItem->get_description();
         $contentLink = $feedItem->get_permalink();
         $date = $feedItem->get_date();
         //Create a new Content item
         $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source);
         //Fill the Content Item
         $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $title, array($description));
         $item->link = $contentLink;
         $item->date = strtotime($date);
         //Add the item to the Content array
         $contentItems[] = $item;
     }
     $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [END: Parsing feed items]", \PEAR_LOG_DEBUG);
     $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Method finished]", \PEAR_LOG_DEBUG);
     //return the content array
     return $contentItems;
 }
Esempio n. 10
0
 /**
  * Given an array of content is's, this function will
  * fetch the content objects from the data store.
  *
  * @param string[] $ids
  * @return \Swiftriver\Core\ObjectModel\Content[]
  */
 public static function GetContent($ids, $orderby = null)
 {
     $logger = \Swiftriver\Core\Setup::GetLogger();
     $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [Method invoked]", \PEAR_LOG_DEBUG);
     //if no $orderby is sent
     if (!$orderby || $orderby == null) {
         $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [No Order By clause set, setting to 'date desc']", \PEAR_LOG_DEBUG);
         //Set it to the default - date DESC
         $orderby = "date desc";
     }
     //set up the return array
     $content = array();
     //If the $ids array is blank or empty, return the empty array
     if (!isset($ids) || !is_array($ids) || count($ids) < 1) {
         $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [No IDs sent to method]", \PEAR_LOG_DEBUG);
         $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [Method finsiehd]", \PEAR_LOG_DEBUG);
         return $content;
     }
     $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [START: Building the RedBean Query]", \PEAR_LOG_DEBUG);
     //set up the array to hold the ids
     $queryIds = array();
     //start to build the sql
     $query = "textId in (";
     /*//for each content item, add to the query and the ids array
       for($i=0; $i<count($ids); $i++) {
           $query .= ":id$i,";
           $queryIds[":id$i"] = $ids[$i];
       }*/
     $counter = 0;
     foreach ($ids as $id) {
         $query .= ":id{$counter},";
         $queryIds[":id{$counter}"] = $id;
         $counter++;
     }
     //tidy up the query
     $query = rtrim($query, ",") . ") order by " . $orderby;
     $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [END: Building the RedBean Query]", \PEAR_LOG_DEBUG);
     $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [START: Running RedBean Query]", \PEAR_LOG_DEBUG);
     //Get the finder
     $finder = RedBeanController::Finder();
     //Find the content
     $dbContent = $finder->where("content", $query, $queryIds);
     $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [FINISHED: Running RedBean Query]", \PEAR_LOG_DEBUG);
     //set up the return array
     $content = array();
     //set up the red bean
     $rb = RedBeanController::RedBean();
     $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [START: Constructing Content and Source items]", \PEAR_LOG_DEBUG);
     //loop through the db content
     foreach ($dbContent as $key => $dbItem) {
         //get the associated source
         $s = reset($rb->batch("source", RedBeanController::GetRelatedBeans($dbItem, "source")));
         //Create the source from the db json
         $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromJSON($s->json);
         //get the json for the content
         $json = $dbItem->json;
         //create the content
         $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source, $json);
         //add it to the array
         $content[] = $item;
     }
     $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [END: Constructing Content and Source items]", \PEAR_LOG_DEBUG);
     $logger->log("Core::Modules::DataContext::MySQL_V1::DataContext::GetContent [Method finished]", \PEAR_LOG_DEBUG);
     //return the content
     return $content;
 }
Esempio n. 11
0
 /**
  * Parses the simplepie item to a content item
  * @param \SimplePie_Item $tweet
  * @param \Swiftriver\Core\ObjectModel\Source
  * @return \Swiftriver\Core\ObjectModel\Content
  */
 private function ParseTweetFromATOMItem($tweet, $channel)
 {
     //Extract all the relevant feedItem info
     $title = $tweet->get_title();
     //$description = $tweet->get_description();
     $contentLink = $tweet->get_permalink();
     $date = $tweet->get_date();
     //Create the source
     $author = $tweet->get_author();
     $source_name = $author != null ? $author->get_name() : $channel->name;
     $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name);
     $source->name = $source_name;
     $source->email = $author != null ? $tweet->get_author()->get_email() : null;
     $source->link = $author != null ? $tweet->get_author()->get_link() : null;
     $source->parent = $channel->id;
     $source->type = $channel->type;
     $source->subType = $channel->subType;
     //Create a new Content item
     $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source);
     //Fill the Content Item
     $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $title, array());
     $item->link = $contentLink;
     $item->date = strtotime($date);
     return $item;
 }
 /**
  * Gets content items from the server
  * @param \Swiftriver\Core\ObjectModel\Channel $channel
  * @param string $serverURL
  *
  * @return array()
  */
 private function getLocalContentItems($channel, $logger)
 {
     $contentItems = array();
     $logger->log("Core::Modules::SiSPS::Parsers::FrontlineSMSParser::getLocalContentItems [Preparing to read from DB]", \PEAR_LOG_DEBUG);
     $lastSuccess = "0";
     if (isset($channel->lastSucess)) {
         $lastSuccess = date('U', $channel->lastSucess) . "000";
     }
     if ($channel->parameters["Database"] == "" && $channel->parameters[""] == "UserName") {
         $logger->log("Core::Modules::SiSPS::Parsers::FrontlineSMSParser::getLocalContentItems [Database and UserName parameters not set, exiting]", \PEAR_LOG_DEBUG);
         return null;
     }
     $my_connection = $this->localDBOpen("localhost", $channel->parameters["UserName"], $channel->parameters["Password"], $channel->parameters["Database"]);
     $my_messages = $this->localDBReturnMessages($my_connection, "message", $lastSuccess);
     $this->localDBClose($my_connection);
     if (is_array($my_messages)) {
         $num_messages = count($my_messages);
         $logger->log("Core::Modules::SiSPS::Parsers::FrontlineSMSParser::getContentItems [Processing {$num_messages} messages]", \PEAR_LOG_DEBUG);
         for ($message_index = 0; $message_index < $num_messages; $message_index++) {
             $source_name = "";
             // Embed source name and source number
             $source_name = $my_messages[$message_index]['senderMsisdn'];
             $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $channel->trusted);
             $source->name = $source_name;
             $source->parent = $channel->id;
             $source->type = $channel->type;
             $source->subType = $channel->subType;
             //Create a new Content item
             $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source);
             $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText("", "", array($my_messages[$message_index]['textContent'], $my_messages[$message_index]['binaryMessageContent']));
             //the message
             $item->link = null;
             $item->date = time();
             $contentItems[] = $item;
         }
         return $contentItems;
     } else {
         return null;
     }
 }
Esempio n. 13
0
 /**
  * Implementation of IParser::GetAndParse
  * @param string[] $parameters
  * Required Parameter Values =
  *  'feedUrl' = The url to the RSS feed
  * @param datetime $lassucess
  */
 public function GetAndParse($parameters, $lastsucess)
 {
     $logger = \Swiftriver\Core\Setup::GetLogger();
     $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Method invoked]", \PEAR_LOG_DEBUG);
     $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [START: Extracting required parameters]", \PEAR_LOG_DEBUG);
     //Extract the required variables
     $feedUrl = $parameters["feedUrl"];
     if (!isset($feedUrl) || $feedUrl == "") {
         $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [the parapeter 'feedUrl' was not supplued. Returning null]", \PEAR_LOG_DEBUG);
         $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG);
         return null;
     }
     $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [END: Extracting required parameters]", \PEAR_LOG_DEBUG);
     $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [START: Constructing source object]", \PEAR_LOG_DEBUG);
     //Create the source that will be used by all the content items Passing in the feed uri which can
     //be used to uniquly identify the source of the content
     $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromID($feedUrl);
     $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [END: Constructing source object]", \PEAR_LOG_DEBUG);
     $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [START: Including the SimplePie module]", \PEAR_LOG_DEBUG);
     //Include the Simple Pie Framework to get and parse feeds
     $config = \Swiftriver\Core\Setup::Configuration();
     include_once $config->ModulesDirectory . "/SimplePie/simplepie.inc";
     $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [END: Including the SimplePie module]", \PEAR_LOG_DEBUG);
     //Construct a new SimplePie Parsaer
     $feed = new \SimplePie();
     //Get the cach directory
     $cacheDirectory = $config->CachingDirectory;
     $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Setting the caching directory to {$cacheDirectory}]", \PEAR_LOG_DEBUG);
     //Set the caching directory
     $feed->set_cache_location($cacheDirectory);
     $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Setting the feed url to {$feedUrl}]", \PEAR_LOG_DEBUG);
     //Pass the feed URL to the SImplePie object
     $feed->set_feed_url($feedUrl);
     $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Initilising the feed]", \PEAR_LOG_DEBUG);
     //Run the SimplePie
     $feed->init();
     //Create the Content array
     $contentItems = array();
     $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [START: Parsing feed items]", \PEAR_LOG_DEBUG);
     $feeditems = $feed->get_items();
     if (!$feeditems || $feeditems == null || !is_array($feeditems) || count($feeditems) < 1) {
         $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [No feeditems recovered from the feed]", \PEAR_LOG_DEBUG);
     }
     //Loop throught the Feed Items
     foreach ($feeditems as $feedItem) {
         //Extract the date of the content
         $contentdate = strtotime($feedItem->get_date());
         if (isset($lastsucess) && is_numeric($lastsucess) && isset($contentdate) && is_numeric($contentdate)) {
             if ($contentdate < $lastsucess) {
                 $textContentDate = date("c", $contentdate);
                 $textLastSucess = date("c", $lastsucess);
                 $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Skipped feed item as date {$textContentDate} less than last sucessful run ({$textLastSucess})]", \PEAR_LOG_DEBUG);
                 continue;
             }
         }
         $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Adding feed item]", \PEAR_LOG_DEBUG);
         //Extract all the relevant feedItem info
         $title = $feedItem->get_title();
         $description = $feedItem->get_description();
         $contentLink = $feedItem->get_permalink();
         $date = $feedItem->get_date();
         //Create a new Content item
         $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source);
         //Fill the Content Item
         $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $title, array($description));
         $item->link = $contentLink;
         $item->date = strtotime($date);
         //Add the item to the Content array
         $contentItems[] = $item;
     }
     $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [END: Parsing feed items]", \PEAR_LOG_DEBUG);
     $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG);
     //return the content array
     return $contentItems;
 }
Esempio n. 14
0
 /**
  * Implementation of IParser::GetAndParse
  * @param \Swiftriver\Core\ObjectModel\Channel $channel
  * @param datetime $lassucess
  */
 public function GetAndParse($channel)
 {
     $logger = \Swiftriver\Core\Setup::GetLogger();
     $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Method invoked]", \PEAR_LOG_DEBUG);
     $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [START: Extracting required parameters]", \PEAR_LOG_DEBUG);
     if ($channel->subType == "Tag Search" || $channel->subType == "Tag Search with Location") {
         $rawTags = $channel->parameters["tags"];
         if (!isset($rawTags) || $rawTags == null) {
             $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [the parameter 'tags' was not supplied. Returning null]", \PEAR_LOG_DEBUG);
             $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG);
             return null;
         }
         $tags = \explode(" ", $rawTags);
         $url = $channel->subType == "Tag Search" ? "http://api.flickr.com/services/feeds/photos_public.gne?format=rss2&tags=" : "http://www.flickr.com/services/feeds/geo?format=rss2&tags=";
         foreach ($tags as $tag) {
             $url .= "{$tag},";
         }
         $url = \rtrim($url, ",");
     } elseif ($channel->subType == "Follow a User") {
         $userid = $channel->parameters["userid"];
         if (!isset($userid) || $userid == null) {
             $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [the parameter 'userid' was not supplied. Returning null]", \PEAR_LOG_DEBUG);
             $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG);
             return null;
         }
         $url = "http://api.flickr.com/services/feeds/photos_public.gne?id={$userid}";
     } else {
         $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [The subType supplied was not recognised]", \PEAR_LOG_ERR);
         $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG);
         return null;
     }
     $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [END: Extracting required parameters]", \PEAR_LOG_DEBUG);
     $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [START: Including the SimplePie module]", \PEAR_LOG_DEBUG);
     //Include the Simple Pie Framework to get and parse feeds
     $config = \Swiftriver\Core\Setup::Configuration();
     $simplePiePath = $config->ModulesDirectory . "/SimplePie/simplepie.inc";
     include_once $simplePiePath;
     //Include the Simple Pie YouTube Framework
     $simpleTubePiePath = $config->ModulesDirectory . "/SimplePie/simpletube.inc";
     include_once $simpleTubePiePath;
     $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [END: Including the SimplePie module]", \PEAR_LOG_DEBUG);
     //Construct a new SimplePie Parser
     $feed = new \SimplePie();
     //Get the cache directory
     $cacheDirectory = $config->CachingDirectory;
     $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Setting the caching directory to {$cacheDirectory}]", \PEAR_LOG_DEBUG);
     //Set the caching directory
     $feed->set_cache_location($cacheDirectory);
     $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Setting the feed url to {$url}]", \PEAR_LOG_DEBUG);
     //Pass the feed URL to the SImplePie object
     $feed->set_feed_url($url);
     $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Initializing the feed]", \PEAR_LOG_DEBUG);
     //Run the SimplePie
     $feed->init();
     //Strip HTML
     $feed->strip_htmltags(array('span', 'font', 'style', 'p'));
     //Create the Content array
     $contentItems = array();
     $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [START: Parsing feed items]", \PEAR_LOG_DEBUG);
     $feeditems = $feed->get_items();
     if (!$feeditems || $feeditems == null || !is_array($feeditems) || count($feeditems) < 1) {
         $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [No feeditems recovered from the feed]", \PEAR_LOG_DEBUG);
     }
     $lastSuccess = $channel->lastSuccess;
     //Loop through the Feed Items
     foreach ($feeditems as $feedItem) {
         //Extract the date of the content
         $contentdate = strtotime($feedItem->get_date());
         if (isset($lastSuccess) && is_numeric($lastSuccess) && isset($contentdate) && is_numeric($contentdate)) {
             if ($contentdate < $lastSuccess) {
                 $textContentDate = date("c", $contentdate);
                 $textlastSuccess = date("c", $lastSuccess);
                 $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Skipped feed item as date {$textContentDate} less than last sucessful run ({$textlastSuccess})]", \PEAR_LOG_DEBUG);
                 continue;
             }
         }
         $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Adding feed item]", \PEAR_LOG_DEBUG);
         //Get source data
         $source_name = $feedItem->get_author()->name;
         if (!isset($source_name) || $source_name == null || $source_name == "") {
             $source_name = "unknown";
         }
         $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $channel->trusted);
         $source->name = $source_name;
         $source->email = $feedItem->get_author()->email;
         $source->parent = $channel->id;
         $source->type = $channel->type;
         $source->subType = $channel->subType;
         //Extract all the relevant feedItem info
         $title = $feedItem->get_title();
         $description = $feedItem->get_description();
         $contentLink = $feedItem->get_permalink();
         $date = $feedItem->get_date();
         $rawLocation = $feedItem->get_item_tags("http://www.georss.org/georss", "point");
         $long = 0;
         $lat = 0;
         $name = "";
         if (is_array($rawLocation)) {
             $lat_lon_array = split(" ", $rawLocation[0]["data"]);
             $long = $lat_lon_array[1];
             $lat = $lat_lon_array[0];
             $location = new \Swiftriver\Core\ObjectModel\GisData($long, $lat, $name);
         }
         //Create a new Content item
         $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source);
         if (isset($location)) {
             $item->gisData = array($location);
         }
         //Fill the Content Item
         $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $title, array($description));
         $item->link = $contentLink;
         $item->date = strtotime($date);
         //Add the item to the Content array
         $contentItems[] = $item;
     }
     $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [END: Parsing feed items]", \PEAR_LOG_DEBUG);
     $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG);
     //return the content array
     return $contentItems;
 }
Esempio n. 15
0
 /**
  * Lists all the current Source in the core
  * @return \Swiftriver\Core\ObjectModel\Source[]
  */
 public static function ListAllSources()
 {
     $logger = \Swiftriver\Core\Setup::GetLogger();
     $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [Method initiated]", \PEAR_LOG_DEBUG);
     $sources = array();
     $selectAllSourcesSql = "CALL SC_SelectAllSources ()";
     try {
         $db = self::PDOConnection();
         $selectAllSourcesStatment = $db->prepare($selectAllSourcesSql);
         $result = $selectAllSourcesStatment->execute();
         if ($result === false) {
             $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [An Exception was thrown by the PDO framwork]", \PEAR_LOG_ERR);
             $errorInfo = $selectAllSourcesStatment->errorInfo();
             $errorMessage = $errorInfo[2];
             $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [{$errorMessage}]", \PEAR_LOG_ERR);
         }
         $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [END: Executing PDO statement]", \PEAR_LOG_DEBUG);
         foreach ($selectAllSourcesStatment->fetchAll() as $row) {
             $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [START: Constructing source obejct]", \PEAR_LOG_DEBUG);
             $json = $row["json"];
             $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromJSON($json);
             $sources[] = $source;
             $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [END: Constructing source obejct]", \PEAR_LOG_DEBUG);
         }
         $db = null;
     } catch (\PDOException $e) {
         $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [An Exception was thrown:]", \PEAR_LOG_ERR);
         $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [{$e}]", \PEAR_LOG_ERR);
     }
     $logger->log("Core::Modules::DataContext::MySQL_V2::DataContext::ListAllSources [Method finished]", \PEAR_LOG_DEBUG);
     return $sources;
 }
Esempio n. 16
0
 /**
  * Parses the simplepie item to a content item
  * @param \SimplePie_Item $tweet
  * @param \Swiftriver\Core\ObjectModel\Channel $channel
  */
 private function ParseTweetFromATOMItem($tweet, $channel)
 {
     //Extract all the relevant feedItem info
     $title = $tweet->get_title();
     //$description = $tweet->get_description();
     $contentLink = $tweet->get_permalink();
     $date = $tweet->get_date();
     //Create the source
     $author = $tweet->get_author();
     $source_name = $author != null ? $author->get_name() : $channel->name;
     $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $channel->trusted);
     $source->name = $source_name;
     $source->email = $author != null ? $tweet->get_author()->get_email() : null;
     $source->link = $author != null ? $tweet->get_author()->get_link() : null;
     $source->parent = $channel->id;
     $source->type = $channel->type;
     $source->subType = $channel->subType;
     //Add location data
     //Long and lat
     $location = $tweet->get_item_tags("http://www.georss.org/georss", "point");
     $long = 0;
     $lat = 0;
     $name = "";
     if (is_array($location)) {
         $lat_lon_array = split(" ", $location[0]["data"]);
         $long = $lat_lon_array[1];
         $lat = $lat_lon_array[0];
         //Name
         $location_name = $tweet->get_item_tags("http://api.twitter.com", "place");
         if (is_array($location_name)) {
             if (isset($location_name[0]["child"]["http://api.twitter.com"]["full_name"][0]["data"])) {
                 $name = $location_name[0]["child"]["http://api.twitter.com"]["full_name"][0]["data"];
             }
         }
         $source->gisData = array(new \Swiftriver\Core\ObjectModel\GisData($long, $lat, $name));
     }
     //Create a new Content item
     $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source);
     //Fill the Content Item
     $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $title, array());
     $item->link = $contentLink;
     $item->date = strtotime($date);
     //Sanitize the tweet text into a DIF collection
     $sanitizedTweetDiffCollection = $this->ParseTweetToSanitizedTweetDiffCollection($tweet);
     //Add the dif collection to the item
     $item->difs = array($sanitizedTweetDiffCollection);
     //return the item
     return $item;
 }
 /**
  * Processes a queue file and does something with it (example only)
  * @param string $queueFile The queue file
  */
 protected function processQueueFile($queueFile)
 {
     $logger = \Swiftriver\Core\Setup::GetLogger();
     $logger->log("Core::Modules::TwitterStreamingSearchClient start processing " . $queueFile, \PEAR_LOG_DEBUG);
     // Open file
     $fp = fopen($queueFile, 'r');
     // Check if something has gone wrong, or perhaps the file is just locked by another process
     if (!is_resource($fp)) {
         $logger->log("Core::Modules::TwitterStreamingSearchClient file: " . $queueFile . " already open, skipping", \PEAR_LOG_DEBUG);
         return FALSE;
     }
     // Lock file
     flock($fp, LOCK_EX);
     $content = array();
     $logger->log("Core::Modules::TwitterStreamingSearchClient: START Looping through content", \PEAR_LOG_DEBUG);
     while ($rawStatus = fgets($fp, 4096)) {
         try {
             $status = \json_decode($rawStatus);
             $source_name = $status->user->screen_name;
             if ($source_name == null || $source_name == "") {
                 $source_name = "UNKNOWN";
             }
             $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, false);
             $source->name = $source_name;
             $source->link = "http://twitter.com/" . $source_name;
             $source->parent = "TWITTERSTREAM";
             $source->type = "Twitter Stream";
             $source->subType = "Filter";
             $source->applicationIds["twitter"] = $status->user->id;
             $source->applicationProfileImages["twitter"] = $status->user->profile_image_url;
             //Create a new Content item
             $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source);
             //Fill the Content Item
             $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $status->text, array($status->text));
             $item->link = "http://twitter.com/" . $source_name . "/statuses/" . $status->id_str;
             $item->date = strtotime($status->created_at);
             /* GEO is not yet supported on this streamin feed
                if($tweet->geo != null && $tweet->geo->type == "Point" && \is_array($tweet->geo->coordinates))
                    $item->gisData[] = new \Swiftriver\Core\ObjectModel\GisData (
                            $tweet->geo->coordinates[1],
                            $tweet->geo->coordinates[0],
                            "");
                */
             //Sanitize the tweet text into a DIF collection
             $sanitizedTweetDiffCollection = $this->ParseTweetToSanitizedTweetDiffCollection($item);
             //Add the dif collection to the item
             $item->difs = array($sanitizedTweetDiffCollection);
             $content[] = $item;
         } catch (\Exception $e) {
             $logger->log("Core::Modules::TwitterStreamingSearchClient: {$e}", \PEAR_LOG_ERR);
         }
     }
     $logger->log("Core::Modules::TwitterStreamingSearchClient: START Looping through content", \PEAR_LOG_DEBUG);
     $workflow = new \Swiftriver\Core\Workflows\ContentServices\ProcessContent();
     //Here we are running the workflow without pre processing.
     $workflow->RunWorkflow($content, false);
     // Release lock and close
     flock($fp, LOCK_UN);
     fclose($fp);
     // All done with this file
     $logger->log("Core::Modules::TwitterStreamingSearchClient finshed processing " . $queueFile, \PEAR_LOG_DEBUG);
     unlink($queueFile);
 }