/** * Returns all the classes that implment the IPreProcessor interface * @return IPreProcessingStep[] */ public function ListAllAvailablePreProcessingSteps() { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::PreProcessing::PreProcessor::ListAllAvailablePreProcessingSteps [Method invoked]", \PEAR_LOG_DEBUG); $steps = array(); $modulesDirectory = \Swiftriver\Core\Setup::Configuration()->ModulesDirectory; $dirItterator = new \RecursiveDirectoryIterator($modulesDirectory); $iterator = new \RecursiveIteratorIterator($dirItterator, \RecursiveIteratorIterator::SELF_FIRST); foreach ($iterator as $file) { if ($file->isFile()) { $filePath = $file->getPathname(); if (strpos($filePath, "PreProcessingStep.php")) { try { include_once $filePath; $typeString = "\\Swiftriver\\PreProcessingSteps\\" . $file->getFilename(); $type = str_replace(".php", "", $typeString); $object = new $type(); if ($object instanceof IPreProcessingStep) { $logger->log("Core::PreProcessing::PreProcessor::ListAllAvailablePreProcessingSteps [Adding type {$type}]", \PEAR_LOG_DEBUG); $object->filePath = str_replace($modulesDirectory, "", $filePath); $object->type = $type; $steps[] = $object; } } catch (\Exception $e) { $logger->log("Core::PreProcessing::PreProcessor::ListAllAvailablePreProcessingSteps [error adding type {$type}]", \PEAR_LOG_DEBUG); $logger->log("Core::PreProcessing::PreProcessor::ListAllAvailablePreProcessingSteps [{$e}]", \PEAR_LOG_ERR); continue; } } } } $logger->log("Core::PreProcessing::PreProcessor::ListAllAvailablePreProcessingSteps [Method finished]", \PEAR_LOG_DEBUG); return $steps; }
/** * Constructor Method * @param string $uri */ public function __construct($uri) { $this->uri = $uri; $this->proxy = \Swiftriver\Core\Setup::Configuration()->ProxyServer; $this->proxyUsername = \Swiftriver\Core\Setup::Configuration()->ProxyServerUserName; $this->proxyPassword = \Swiftriver\Core\Setup::Configuration()->ProxyServerPassword; }
public static function Toolbox() { //if not the first time this is called if (isset(self::$toolbox)) { return self::$toolbox; } //else set up the RedBean toolbox require_once \Swiftriver\Core\Setup::Configuration()->ModulesDirectory . "/RedBean/redbean.inc.php"; //get the url of the url of the db server $url = (string) Setup::$Configuration->DataBaseUrl; //Get the db username $username = (string) Setup::$Configuration->UserName; //get the password $password = (string) Setup::$Configuration->Password; //get the db name $database = (string) Setup::$Configuration->Database; //set the db type $typeofdatabase = "mysql"; //Assemble a database connection string (DSN) $dsn = "{$typeofdatabase}:host={$url};dbname={$database}"; //Construct a new Red Bean Toolbox, if it has not been set in the mean time if (!isset(self::$toolbox)) { self::$toolbox = \RedBean_Setup::kickstartDev($dsn, $username, $password); } //return it return self::$toolbox; }
public function RunWorkflow($key) { $filename = \Swiftriver\Core\Setup::Configuration()->CachingDirectory . "/TwitterStreamingController.go"; if (!\file_exists($filename)) { return parent::FormatReturn('{"IsActive":false}'); } return parent::FormatReturn('{"IsActive":true}'); }
public function RunWorkflow($key) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::ServiceAPI::TwitterStreamingServices::StopTwitterStreamer::RunWorkflow [Method invoked]", \PEAR_LOG_INFO); $filename = \Swiftriver\Core\Setup::Configuration()->CachingDirectory . "/TwitterStreamingController.go"; if (\file_exists($filename)) { \unlink($filename); } $logger->log("Core::ServiceAPI::TwitterStreamingServices::StopTwitterStreamer::RunWorkflow [Method finished]", \PEAR_LOG_INFO); return '{"message":"OK"}'; }
/** * Returns all the classes that implment the IPreProcessor interface * * @return IPreProcessingStep[] */ public function ListAllAvailablePreProcessingSteps() { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::PreProcessing::PreProcessor::ListAllAvailablePreProcessingSteps [Method invoked]", \PEAR_LOG_DEBUG); //set up the array to hold the steps $steps = array(); //Get the path of the modules directory $modulesDirectory = \Swiftriver\Core\Setup::Configuration()->ModulesDirectory; //Create a recursive directory ittorator $dirItterator = new \RecursiveDirectoryIterator($modulesDirectory); $iterator = new \RecursiveIteratorIterator($dirItterator, \RecursiveIteratorIterator::SELF_FIRST); //Itterate over all the files in all the directories foreach ($iterator as $file) { //If not a file continue if (!$file->isFile()) { continue; } //get the full file path $filePath = $file->getPathname(); //If not a pre processing step then continue if (!strpos($filePath, "PreProcessingStep.php")) { continue; } try { //Include the file include_once $filePath; //create a type string for the pre processing step $typeString = "\\Swiftriver\\PreProcessingSteps\\" . $file->getFilename(); //remove the .php extension $type = str_replace(".php", "", $typeString); //instanciate the pre processing step $object = new $type(); //Check that the object implements the IPreProcessingStep if (!$object instanceof IPreProcessingStep) { continue; } $logger->log("Core::PreProcessing::PreProcessor::ListAllAvailablePreProcessingSteps [Adding type {$type}]", \PEAR_LOG_DEBUG); //Add the file name to the step element $object->filePath = str_replace($modulesDirectory, "", $filePath); //Add the type to the step element $object->type = $type; //Add the object to the array $steps[] = $object; } catch (\Exception $e) { $logger->log("Core::PreProcessing::PreProcessor::ListAllAvailablePreProcessingSteps [error adding type {$type}]", \PEAR_LOG_DEBUG); $logger->log("Core::PreProcessing::PreProcessor::ListAllAvailablePreProcessingSteps [{$e}]", \PEAR_LOG_ERR); continue; } } $logger->log("Core::PreProcessing::PreProcessor::ListAllAvailablePreProcessingSteps [Method finished]", \PEAR_LOG_DEBUG); return $steps; }
/** * Implementation of IParser::GetAndParse * @param \Swiftriver\Core\ObjectModel\Channel $channel * @param datetime $lassucess */ public function GetAndParse($channel) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::Modules::SiSPS::Parsers::FeedsParser::GetAndParse [Method invoked]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::FeedsParser::GetAndParse [START: Extracting required parameters]", \PEAR_LOG_DEBUG); //Extract the required variables $feedUrl = ""; switch ($channel->subType) { case "Wordpress Blog": $feedUrl = $channel->parameters["BlogUrl"]; $feedUrl = rtrim($feedUrl, "/") . "/?feed=atom"; break; case "Blogger Blog": $feedUrl = $channel->parameters["BlogUrl"]; $feedUrl = rtrim($feedUrl, "/") . "/feeds/posts/default"; break; case "Tumblr Blog": $feedUrl = $channel->parameters["BlogUrl"]; $feedUrl = rtrim($feedUrl, "/") . "/api/read"; break; case "News Feeds": $feedUrl = $channel->parameters["feedUrl"]; break; } if (!isset($feedUrl) || $feedUrl == "") { $logger->log("Core::Modules::SiSPS::Parsers::FeedsParser::GetAndParse [the parameter 'feedUrl' was not supplied. Returning null]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::FeedsParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG); return null; } $logger->log("Core::Modules::SiSPS::Parsers::FeedsParser::GetAndParse [END: Extracting required parameters]", \PEAR_LOG_DEBUG); //Include the Simple Pie Framework to get and parse feeds $config = \Swiftriver\Core\Setup::Configuration(); $contentItems = null; switch ($channel->subType) { case "Wordpress Blog": $contentItems = $this->GetSimplePieContentEntries($channel, $config, $logger, $feedUrl); break; case "Blogger Blog": $contentItems = $this->GetSimplePieContentEntries($channel, $config, $logger, $feedUrl); break; case "Tumblr Blog": $contentItems = $this->GetXMLContentEntries($channel, $config, $logger, $feedUrl); break; case "News Feeds": $contentItems = $this->GetSimplePieContentEntries($channel, $config, $logger, $feedUrl); break; } //return the content array return $contentItems; }
public function RunWorkflow($key) { $filename = \Swiftriver\Core\Setup::Configuration()->CachingDirectory . "/TwitterStreamingController.tmp"; if (!\file_exists($filename)) { return parent::FormatMessage("no config"); } $fp = fopen($filename, "r+"); $line = fread($fp, filesize($filename)); fclose($fp); $parts = explode("|", $line); $object->TwitterUsername = $parts[0]; $object->TwitterPassword = $parts[1]; $object->SearchTerms = explode("~", $parts[2]); return parent::FormatReturn(\json_encode($object)); }
/** * Implementation of IParser::GetAndParse * @param string[][] $parameters * Required Parameter Values = * 'feedUrl' = The url to the RSS feed */ public function GetAndParse($parameters) { //Extract the required variables $feedUrl = $parameters["feedUrl"]; if (!isset($feedUrl) || $feedUrl == "") { return null; } //Create the source that will be used by all the content items //Passing in the feed uri which can be used to uniquly //identify the source of the content $source = new \Swiftriver\Core\ObjectModel\Source($feedUrl); //Include the Simple Pie Framework to get and parse feeds $config = \Swiftriver\Core\Setup::Configuration(); include_once $config->ModulesDirectory . "/SimplePie/simplepie.inc"; //Construct a new SimplePie Parsaer $feed = new \SimplePie(); //Set the caching directory $feed->set_cache_location($config->CachingDirectory); //Pass the feed URL to the SImplePie object $feed->set_feed_url($feedUrl); //Run the SimplePie $feed->init(); //Create the Content array $contentItems = array(); //Loop throught the Feed Items foreach ($feed->get_items() as $feedItem) { //Extract all the relevant feedItem info $id = \Swiftriver\Core\ObjectModel\Content::GenerateUniqueId(); $title = $feedItem->get_title(); $description = $feedItem->get_description(); $contentLink = $feedItem->get_permalink(); //Create a new Content item $item = new \Swiftriver\Core\ObjectModel\Content(); //Fill the COntenty Item $item->SetId($id); $item->SetTitle($title); $item->SetLink($contentLink); $item->SetText(array($description)); $item->SetSource($source); //Add the item to the Content array $contentItems[] = $item; } //return the content array return $contentItems; }
public function checkFilterPredicates() { $filename = \Swiftriver\Core\Setup::Configuration()->CachingDirectory . "/TwitterStreamingController.go"; if (!\file_exists($filename)) { parent::disconnect(); } $sec = (int) date('s'); //this is called every 5 secs so to give us a break we ease off in the last 30 seconds of every minute if ($sec > 30) { return; } $logger = \Swiftriver\Core\Setup::GetLogger(); $queueFiles = glob($this->queueDir . '/phirehose-queue*.queue'); $logger->log("Core::Modules::TwitterStreamingSearchClient Found " . count($queueFiles) . " queue files.", \PEAR_LOG_DEBUG); foreach ($queueFiles as $queueFile) { $this->processQueueFile($queueFile); } }
public function PreProcessContent($content) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::PreProcessing::PreProcessor::PreProcessContent [Method invoked]", \PEAR_LOG_DEBUG); $modulesDirectory = \Swiftriver\Core\Setup::Configuration()->ModulesDirectory; $configuration = \Swiftriver\Core\Setup::Configuration(); if (isset($this->preProcessingSteps) && count($this->preProcessingSteps) > 0) { foreach ($this->preProcessingSteps as $preProcessingStep) { //Get the class name from config $className = $preProcessingStep->className; //get the file path from config $filePath = $modulesDirectory . $preProcessingStep->filePath; $logger->log("Core::PreProcessing::PreProcessor::PreProcessContent [START: Including pre processor: {$filePath}]", \PEAR_LOG_DEBUG); //Include the file include_once $filePath; $logger->log("Core::PreProcessing::PreProcessor::PreProcessContent [END: Including pre processor: {$filePath}]", \PEAR_LOG_DEBUG); $logger->log("Core::PreProcessing::PreProcessor::PreProcessContent [START: Instanciating pre processor: {$className}]", \PEAR_LOG_DEBUG); try { //Instanciate the pre processor $preProcessor = new $className(); } catch (\Exception $e) { $message = $e->getMessage(); $logger->log("Core::PreProcessing::PreProcessor::PreProcessContent [{$message}]", \PEAR_LOG_ERR); $logger->log("Core::PreProcessing::PreProcessor::PreProcessContent [Unable to run PreProcessing for preprocessor {$className}]", \PEAR_LOG_ERR); } $logger->log("Core::PreProcessing::PreProcessor::PreProcessContent [END: Instanciating pre processor: {$className}]", \PEAR_LOG_DEBUG); $logger->log("Core::PreProcessing::PreProcessor::PreProcessContent [START: Run PreProcessing for {$className}]", \PEAR_LOG_DEBUG); try { //Run the preocess method on the pre processor $content = $preProcessor->Process($content, $configuration, $logger); } catch (\Exception $e) { $message = $e->getMessage(); $logger->log("Core::PreProcessing::PreProcessor::PreProcessContent [{$message}]", \PEAR_LOG_ERR); $logger->log("Core::PreProcessing::PreProcessor::PreProcessContent [Unable to run PreProcessing for preprocessor {$className}]", \PEAR_LOG_ERR); } $logger->log("Core::PreProcessing::PreProcessor::PreProcessContent [END: Run PreProcessing for {$className}]", \PEAR_LOG_DEBUG); } } else { $logger->log("Core::PreProcessing::PreProcessor::PreProcessContent [No PreProcessing Steps found to run]", \PEAR_LOG_DEBUG); } $logger->log("Core::PreProcessing::PreProcessor::PreProcessContent [Method finished]", \PEAR_LOG_DEBUG); //Return the content return $content; }
public function RaiseAndDistributeEvent($event) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::EventDistribution::EventDistributor [Method invoked]", \PEAR_LOG_DEBUG); $modulesDirectory = \Swiftriver\Core\Setup::Configuration()->ModulesDirectory; $configuration = \Swiftriver\Core\Setup::Configuration(); if (isset($this->eventHandlers) && count($this->eventHandlers) > 0) { foreach ($this->eventHandlers as $eventHandler) { //Get the class name from config $className = $eventHandler->className; //get the file path from config $filePath = $modulesDirectory . $eventHandler->filePath; $logger->log("Core::EventDistribution::EventDistributor::RaiseAndDistributeEvent [START: Including event handler: {$filePath}]", \PEAR_LOG_DEBUG); //Include the file include_once $filePath; $logger->log("Core::EventDistribution::EventDistributor::RaiseAndDistributeEvent [END: Including event handler: {$filePath}]", \PEAR_LOG_DEBUG); $logger->log("Core::EventDistribution::EventDistributor::RaiseAndDistributeEvent [START: Instanciating event handler: {$className}]", \PEAR_LOG_DEBUG); try { //Instanciate the event handlerr $handler = new $className(); } catch (\Exception $e) { $message = $e->getMessage(); $logger->log("Core::EventDistribution::EventDistributor::RaiseAndDistributeEvent [{$message}]", \PEAR_LOG_ERR); $logger->log("Core::EventDistribution::EventDistributor::RaiseAndDistributeEvent [Unable to run event distribution for event handler: {$className}]", \PEAR_LOG_ERR); } $logger->log("Core::EventDistribution::EventDistributor::RaiseAndDistributeEvent [END: Instanciating event handler: {$className}]", \PEAR_LOG_DEBUG); $logger->log("Core::EventDistribution::EventDistributor::RaiseAndDistributeEvent [START: Run event distribution for {$className}]", \PEAR_LOG_DEBUG); try { //Run the handle event method $handler->HandleEvent($event, $configuration, $logger); } catch (\Exception $e) { $message = $e->getMessage(); $logger->log("Core::EventDistribution::EventDistributor::RaiseAndDistributeEvent [{$message}]", \PEAR_LOG_ERR); $logger->log("Core::EventDistribution::EventDistributor::RaiseAndDistributeEvent [Unable to run event distribution for event handler: {$className}]", \PEAR_LOG_ERR); } $logger->log("Core::EventDistribution::EventDistributor::RaiseAndDistributeEvent [END: Run event distribution for {$className}]", \PEAR_LOG_DEBUG); } } else { $logger->log("Core::EventDistribution::EventDistributor::RaiseAndDistributeEvent [No event handlers found to run]", \PEAR_LOG_DEBUG); } $logger->log("Core::EventDistribution::EventDistributor::RaiseAndDistributeEvent [Method finished]", \PEAR_LOG_DEBUG); }
public function __construct($modulesDirectory = null) { if ($modulesDirectory == null) { $modulesDirectory = \Swiftriver\Core\Setup::Configuration()->ModulesDirectory; } $this->preProcessingSteps = array(); $dirItterator = new \RecursiveDirectoryIterator($modulesDirectory); $iterator = new \RecursiveIteratorIterator($dirItterator, \RecursiveIteratorIterator::SELF_FIRST); foreach ($iterator as $file) { if ($file->isFile()) { $filePath = $file->getPathname(); if (strpos($filePath, "PreProcessingStep")) { include_once $filePath; $className = str_replace(".php", "", $file->getFilename()); $className = "\\Swiftriver\\PreProcessingSteps\\" . $className; $this->preProcessingSteps[] = new $className(); } } } }
public function RunWorkflow($json, $key) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::ServiceAPI::TwitterStreamingServices::StartTwitterStreamer::RunWorkflow [Method invoked]", \PEAR_LOG_INFO); $parameters = parent::ParseTwitterStreamingStartParametersFromJson($json); if (!isset($parameters)) { $logger->log("Core::ServiceAPI::TwitterStreamingServices::StartTwitterStreamer::RunWorkflow [ERROR: Method ParseTwitterStreamingStartParametersFromJson returned null]", \PEAR_LOG_DEBUG); parent::FormatErrorMessage("There was an error in the JSON supplied, please consult the API documentation and try again."); } //create a channel for the streamed content $channelJson = '{"id":"TWITTERSTREAM","type":"Twitter Stream","subType":"Filter","name":"Twitter Stream","active":false}'; $channel = \Swiftriver\Core\ObjectModel\ObjectFactories\ChannelFactory::CreateChannelFromJSON($channelJson); $channelRepository = new \Swiftriver\Core\DAL\Repositories\ChannelRepository(); $channelRepository->SaveChannels(array($channel)); $filename = \Swiftriver\Core\Setup::Configuration()->CachingDirectory . "/TwitterStreamingController.tmp"; $fp = \fopen($filename, "w"); $done = false; while (!$done) { if (\flock($fp, \LOCK_EX)) { $searchArray = ""; foreach ($parameters["SearchTerms"] as $term) { $searchArray .= $term . "~"; } $searchArray = \rtrim($searchArray, '~'); \fwrite($fp, $parameters["TwitterUsername"] . "|" . $parameters["TwitterPassword"] . "|" . $searchArray); \flock($fp, \LOCK_UN); $done = true; } else { \sleep(1); } } \fclose($fp); $command = "php " . \dirname(__FILE__) . "/../../Modules/Phirehose/starttwitterstreamer.php"; $this->execInBackground($command); $logger->log("Core::ServiceAPI::TwitterStreamingServices::StartTwitterStreamer::RunWorkflow [Method finished]", \PEAR_LOG_INFO); return; }
/** * User the twitter RSS call to follow the tweets of a given * twitter user. * * @param \Swiftriver\Core\ObjectModel\Source $source * @return \Swiftriver\Core\ObjectModel\Content[] */ private function GetForTwitterAccount($channel) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::Modules::SiSPS::Parsers::TwitterParser::GetForTwitterAccount [Method invoked]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::TwitterParser::GetForTwitterAccount [START: Extracting required parameters]", \PEAR_LOG_DEBUG); //Extract the required variables $TwitterAccount = $channel->parameters["TwitterAccount"]; if (!isset($TwitterAccount) || $TwitterAccount == "") { $logger->log("Core::Modules::SiSPS::Parsers::TwitterParser::GetForTwitterAccount [the parapeter 'TwitterAccount' was not supplued. Returning null]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::TwitterParser::GetForTwitterAccount [Method finished]", \PEAR_LOG_DEBUG); return null; } $logger->log("Core::Modules::SiSPS::Parsers::TwitterParser::GetForTwitterAccount [END: Extracting required parameters]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::TwitterParser::GetForTwitterAccount [START: Including the SimplePie module]", \PEAR_LOG_DEBUG); //Include the Simple Pie Framework to get and parse feeds $config = \Swiftriver\Core\Setup::Configuration(); include_once $config->ModulesDirectory . "/SimplePie/simplepie.inc"; $logger->log("Core::Modules::SiSPS::Parsers::TwitterParser::GetForTwitterAccount [END: Including the SimplePie module]", \PEAR_LOG_DEBUG); //Construct a new SimplePie Parsaer $feed = new \SimplePie(); //Get the cach directory $cacheDirectory = $config->CachingDirectory; $logger->log("Core::Modules::SiSPS::Parsers::TwitterParser::GetForTwitterAccount [Setting the caching directory to {$cacheDirectory}]", \PEAR_LOG_DEBUG); //Set the caching directory $feed->set_cache_location($cacheDirectory); //Twitter url combined with the account name passed to this feed. $TwitterUrl = "http://twitter.com/statuses/user_timeline/" . $TwitterAccount . ".rss"; //Pass the feed URL to the SImplePie object $feed->set_feed_url($TwitterUrl); $logger->log("Core::Modules::SiSPS::Parsers::TwitterParser::GetForTwitterAccount [Setting the feed url to {$TwitterUrl}]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::TwitterParser::GetForTwitterAccount [Initilising the feed]", \PEAR_LOG_DEBUG); $feed->enable_cache(false); //Run the SimplePie $feed->init(); //Create the Content array $contentItems = array(); $logger->log("Core::Modules::SiSPS::Parsers::TwitterParser::GetForTwitterAccount [START: Parsing feed items]", \PEAR_LOG_DEBUG); $tweets = $feed->get_items(); if (!$tweets || $tweets == null || !is_array($tweets) || count($tweets) < 1) { $logger->log("Core::Modules::SiSPS::Parsers::TwitterParser::GetForTwitterAccount [No feeditems recovered from the feed]", \PEAR_LOG_DEBUG); } //Loop throught the Feed Items foreach ($tweets as $tweet) { //Extract the date of the content $contentdate = strtotime($tweet->get_date('c')); if (isset($channel->lastSuccess) && is_numeric($channel->lastSuccess) && isset($contentdate) && is_numeric($contentdate)) { if ($contentdate < $channel->lastSuccess) { $textContentDate = date("c", $contentdate); $textlastSuccess = date("c", $channel->lastSuccess); $logger->log("Core::Modules::SiSPS::Parsers::TwitterParser::GetForTwitterAccount [Skipped feed item as date {$textContentDate} less than last sucessful run ({$textlastSuccess})]", \PEAR_LOG_DEBUG); continue; } } $logger->log("Core::Modules::SiSPS::Parsers::TwitterParser::GetForTwitterAccount [Adding feed item]", \PEAR_LOG_DEBUG); //Extract all the relevant feedItem info $item = $this->ParseTweetFromATOMItem($tweet, $channel); //Add the item to the Content array $contentItems[] = $item; } $logger->log("Core::Modules::SiSPS::Parsers::TwitterParser::GetForTwitterAccount [END: Parsing feed items]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::TwitterParser::GetForTwitterAccount [Method finished]", \PEAR_LOG_DEBUG); //return the content array return $contentItems; }
<?php include_once dirname(__FILE__) . "/../../Setup.php"; include_once dirname(__FILE__) . "/TwitterStreamingSearchClient.php"; $filename = \Swiftriver\Core\Setup::Configuration()->CachingDirectory . "/TwitterStreamingController.tmp"; $fp = fopen($filename, "r+"); $line = fread($fp, filesize($filename)); fclose($fp); $parts = explode("|", $line); $searchterms = explode("~", $parts[2]); $gofile = str_replace(".tmp", ".go", $filename); //delete the old go file - this will cause any existing streams to cancel if (file_exists($gofile)) { unlink($gofile); } //wait for the existing stream to quit sleep(10); $fp = fopen($gofile, "w"); fclose($fp); //start the new stream $c = new \Swiftriver\Core\Modules\TwitterStreamingSearchClient($parts[0], $parts[1]); $c->setTrack($searchterms); $c->consume();
/** * Constructor for the Analytics engine that can take * an optional modules directory. * * @param string $modulesDirectory */ public function __construct($modulesDirectory = null) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::Analytics::AnalyticsEngine::__construct [Method invoked]", \PEAR_LOG_DEBUG); if ($modulesDirectory == null) { $modulesDirectory = \Swiftriver\Core\Setup::Configuration()->ModulesDirectory; } $modulesDirectory = \rtrim($modulesDirectory, "/") . "/"; $dataContextType = \Swiftriver\Core\Setup::DALConfiguration()->DataContextType; $this->analyticsProviders = array(); $logger->log("Core::Analytics::AnalyticsEngine::__construct [START: Looking for IAnalyticsProviders]", \PEAR_LOG_DEBUG); try { $handle = \opendir($modulesDirectory); while (false !== ($resource = \readdir($handle))) { if (\strpos($resource, "AnalyticsProvider") === false) { continue; } $dirName = $modulesDirectory . $resource . "/"; if (!\is_dir($dirName)) { continue; } $innerHandle = \opendir($dirName); $logger->log("Core::Analytics::AnalyticsEngine::__construct [START: Looking for IAnalyticsProviders in directory {$dirName}]", \PEAR_LOG_DEBUG); while (false !== ($innerResource = \readdir($innerHandle))) { $fileName = $dirName . $innerResource; if (!\is_file($fileName)) { continue; } if (\strpos($innerResource, "AnalyticsProvider.php") === false) { continue; } try { $logger->log("Core::Analytics::AnalyticsEngine::__construct [START: Including the provider {$fileName}]", \PEAR_LOG_DEBUG); include_once $fileName; $logger->log("Core::Analytics::AnalyticsEngine::__construct [END: Including the provider {$fileName}]", \PEAR_LOG_DEBUG); $className = \str_replace(".php", "", $innerResource); $type = "\\Swiftriver\\AnalyticsProviders\\" . $className; $logger->log("Core::Analytics::AnalyticsEngine::__construct [START: Instanciating the provider {$type}]", \PEAR_LOG_DEBUG); $instance = new $type(); $logger->log("Core::Analytics::AnalyticsEngine::__construct [END: Instanciating the provider {$type}]", \PEAR_LOG_DEBUG); if (!$instance instanceof IAnalyticsProvider) { continue; } if (!\in_array($dataContextType, $instance->DataContentSet())) { continue; } $logger->log("Core::Analytics::AnalyticsEngine::__construct [START: Adding the provider {$type}]", \PEAR_LOG_DEBUG); $this->analyticsProviders[] = $instance; $logger->log("Core::Analytics::AnalyticsEngine::__construct [END: Adding the provider {$type}]", \PEAR_LOG_DEBUG); } catch (\Exception $innerE) { $logger->log("Core::Analytics::AnalyticsEngine::__construct [An exception was thrown]", \PEAR_LOG_ERR); $logger->log("Core::Analytics::AnalyticsEngine::__construct [{$innerE}]", \PEAR_LOG_ERR); } } $logger->log("Core::Analytics::AnalyticsEngine::__construct [END: Looking for IAnalyticsProviders in directory {$dirName}]", \PEAR_LOG_DEBUG); } } catch (\Exception $e) { $logger->log("Core::Analytics::AnalyticsEngine::__construct [An exception was thrown]", \PEAR_LOG_ERR); $logger->log("Core::Analytics::AnalyticsEngine::__construct [{$e}]", \PEAR_LOG_ERR); } $logger->log("Core::Analytics::AnalyticsEngine::__construct [END: Looking for IAnalyticsProviders]", \PEAR_LOG_DEBUG); $logger->log("Core::Analytics::AnalyticsEngine::__construct [Method finished]", \PEAR_LOG_DEBUG); }
function GetSimplePieContentEntries($feedUrl, $logger, $channel) { //Include the Simple Pie Framework to get and parse feeds $config = \Swiftriver\Core\Setup::Configuration(); $simplePiePath = $config->ModulesDirectory . "/SimplePie/simplepie.inc"; include_once $simplePiePath; //Include the Simple Pie YouTube Framework $simpleTubePiePath = $config->ModulesDirectory . "/SimplePie/simpletube.inc"; include_once $simpleTubePiePath; $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [END: Including the SimplePie module]", \PEAR_LOG_DEBUG); //Construct a new SimplePie Parser $feed = new \SimplePie(); //Get the cache directory $cacheDirectory = $config->CachingDirectory; $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Setting the caching directory to {$cacheDirectory}]", \PEAR_LOG_DEBUG); //Set the caching directory $feed->set_cache_location($cacheDirectory); $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Setting the feed url to {$feedUrl}]", \PEAR_LOG_DEBUG); //Pass the feed URL to the SImplePie object $feed->set_feed_url($feedUrl); $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Initializing the feed]", \PEAR_LOG_DEBUG); //Run the SimplePie $feed->init(); //Strip HTML $feed->strip_htmltags(array('span', 'font', 'style', 'table', 'td', 'tr', 'div', 'p', 'br', 'a')); //Create the Content array $contentItems = array(); $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [START: Parsing feed items]", \PEAR_LOG_DEBUG); $feeditems = $feed->get_items(); if (!$feeditems || $feeditems == null || !is_array($feeditems) || count($feeditems) < 1) { $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [No feeditems recovered from the feed]", \PEAR_LOG_DEBUG); } $lastSuccess = $channel->lastSuccess; //Loop through the Feed Items foreach ($feeditems as $feedItem) { //Extract the date of the content $contentdate = strtotime($feedItem->get_date()); if (isset($lastSuccess) && is_numeric($lastSuccess) && isset($contentdate) && is_numeric($contentdate)) { if ($contentdate < $lastSuccess) { $textContentDate = date("c", $contentdate); $textlastSuccess = date("c", $lastSuccess); $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Skipped feed item as date {$textContentDate} less than last sucessful run ({$textlastSuccess})]", \PEAR_LOG_DEBUG); continue; } } $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Adding feed item]", \PEAR_LOG_DEBUG); //Get source data $source_name = $feedItem->get_author()->name; $source_name = $source_name == null || $source_name == "" ? "Google News Search -" . $this->searchPhrase : $source_name . " @ " . "Google News Search - " . $this->searchPhrase; $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $channel->trusted); $source->name = $source_name; $source->email = $feedItem->get_author()->email; $source->parent = $channel->id; $source->type = $channel->type; $source->subType = $channel->subType; //Extract all the relevant feedItem info $title = $feedItem->get_title(); $description = $feedItem->get_description(); $contentLink = $feedItem->get_permalink(); $date = $feedItem->get_date(); //Create a new Content item $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source); //Fill the Content Item $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $title, array($description)); $item->link = $contentLink; $item->date = strtotime($date); //Add the item to the Content array $contentItems[] = $item; } $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [END: Parsing feed items]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::GoogleNewsParser::GetSimplePieContentEntries [Method finished]", \PEAR_LOG_DEBUG); //return the content array return $contentItems; }
/** * Implementation of IParser::GetAndParse * @param string[] $parameters * Required Parameter Values = * 'feedUrl' = The url to the RSS feed * @param datetime $lassucess */ public function GetAndParse($parameters, $lastsucess) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Method invoked]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [START: Extracting required parameters]", \PEAR_LOG_DEBUG); //Extract the required variables $feedUrl = $parameters["feedUrl"]; if (!isset($feedUrl) || $feedUrl == "") { $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [the parapeter 'feedUrl' was not supplued. Returning null]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG); return null; } $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [END: Extracting required parameters]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [START: Constructing source object]", \PEAR_LOG_DEBUG); //Create the source that will be used by all the content items Passing in the feed uri which can //be used to uniquly identify the source of the content $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromID($feedUrl); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [END: Constructing source object]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [START: Including the SimplePie module]", \PEAR_LOG_DEBUG); //Include the Simple Pie Framework to get and parse feeds $config = \Swiftriver\Core\Setup::Configuration(); include_once $config->ModulesDirectory . "/SimplePie/simplepie.inc"; $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [END: Including the SimplePie module]", \PEAR_LOG_DEBUG); //Construct a new SimplePie Parsaer $feed = new \SimplePie(); //Get the cach directory $cacheDirectory = $config->CachingDirectory; $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Setting the caching directory to {$cacheDirectory}]", \PEAR_LOG_DEBUG); //Set the caching directory $feed->set_cache_location($cacheDirectory); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Setting the feed url to {$feedUrl}]", \PEAR_LOG_DEBUG); //Pass the feed URL to the SImplePie object $feed->set_feed_url($feedUrl); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Initilising the feed]", \PEAR_LOG_DEBUG); //Run the SimplePie $feed->init(); //Create the Content array $contentItems = array(); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [START: Parsing feed items]", \PEAR_LOG_DEBUG); $feeditems = $feed->get_items(); if (!$feeditems || $feeditems == null || !is_array($feeditems) || count($feeditems) < 1) { $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [No feeditems recovered from the feed]", \PEAR_LOG_DEBUG); } //Loop throught the Feed Items foreach ($feeditems as $feedItem) { //Extract the date of the content $contentdate = strtotime($feedItem->get_date()); if (isset($lastsucess) && is_numeric($lastsucess) && isset($contentdate) && is_numeric($contentdate)) { if ($contentdate < $lastsucess) { $textContentDate = date("c", $contentdate); $textLastSucess = date("c", $lastsucess); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Skipped feed item as date {$textContentDate} less than last sucessful run ({$textLastSucess})]", \PEAR_LOG_DEBUG); continue; } } $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Adding feed item]", \PEAR_LOG_DEBUG); //Extract all the relevant feedItem info $title = $feedItem->get_title(); $description = $feedItem->get_description(); $contentLink = $feedItem->get_permalink(); $date = $feedItem->get_date(); //Create a new Content item $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source); //Fill the Content Item $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $title, array($description)); $item->link = $contentLink; $item->date = strtotime($date); //Add the item to the Content array $contentItems[] = $item; } $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [END: Parsing feed items]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::RSSParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG); //return the content array return $contentItems; }
/** * Implementation of IParser::GetAndParse * @param \Swiftriver\Core\ObjectModel\Channel $channel * @param datetime $lassucess */ public function GetAndParse($channel) { $logger = \Swiftriver\Core\Setup::GetLogger(); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Method invoked]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [START: Extracting required parameters]", \PEAR_LOG_DEBUG); if ($channel->subType == "Tag Search" || $channel->subType == "Tag Search with Location") { $rawTags = $channel->parameters["tags"]; if (!isset($rawTags) || $rawTags == null) { $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [the parameter 'tags' was not supplied. Returning null]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG); return null; } $tags = \explode(" ", $rawTags); $url = $channel->subType == "Tag Search" ? "http://api.flickr.com/services/feeds/photos_public.gne?format=rss2&tags=" : "http://www.flickr.com/services/feeds/geo?format=rss2&tags="; foreach ($tags as $tag) { $url .= "{$tag},"; } $url = \rtrim($url, ","); } elseif ($channel->subType == "Follow a User") { $userid = $channel->parameters["userid"]; if (!isset($userid) || $userid == null) { $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [the parameter 'userid' was not supplied. Returning null]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG); return null; } $url = "http://api.flickr.com/services/feeds/photos_public.gne?id={$userid}"; } else { $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [The subType supplied was not recognised]", \PEAR_LOG_ERR); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG); return null; } $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [END: Extracting required parameters]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [START: Including the SimplePie module]", \PEAR_LOG_DEBUG); //Include the Simple Pie Framework to get and parse feeds $config = \Swiftriver\Core\Setup::Configuration(); $simplePiePath = $config->ModulesDirectory . "/SimplePie/simplepie.inc"; include_once $simplePiePath; //Include the Simple Pie YouTube Framework $simpleTubePiePath = $config->ModulesDirectory . "/SimplePie/simpletube.inc"; include_once $simpleTubePiePath; $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [END: Including the SimplePie module]", \PEAR_LOG_DEBUG); //Construct a new SimplePie Parser $feed = new \SimplePie(); //Get the cache directory $cacheDirectory = $config->CachingDirectory; $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Setting the caching directory to {$cacheDirectory}]", \PEAR_LOG_DEBUG); //Set the caching directory $feed->set_cache_location($cacheDirectory); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Setting the feed url to {$url}]", \PEAR_LOG_DEBUG); //Pass the feed URL to the SImplePie object $feed->set_feed_url($url); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Initializing the feed]", \PEAR_LOG_DEBUG); //Run the SimplePie $feed->init(); //Strip HTML $feed->strip_htmltags(array('span', 'font', 'style', 'p')); //Create the Content array $contentItems = array(); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [START: Parsing feed items]", \PEAR_LOG_DEBUG); $feeditems = $feed->get_items(); if (!$feeditems || $feeditems == null || !is_array($feeditems) || count($feeditems) < 1) { $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [No feeditems recovered from the feed]", \PEAR_LOG_DEBUG); } $lastSuccess = $channel->lastSuccess; //Loop through the Feed Items foreach ($feeditems as $feedItem) { //Extract the date of the content $contentdate = strtotime($feedItem->get_date()); if (isset($lastSuccess) && is_numeric($lastSuccess) && isset($contentdate) && is_numeric($contentdate)) { if ($contentdate < $lastSuccess) { $textContentDate = date("c", $contentdate); $textlastSuccess = date("c", $lastSuccess); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Skipped feed item as date {$textContentDate} less than last sucessful run ({$textlastSuccess})]", \PEAR_LOG_DEBUG); continue; } } $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Adding feed item]", \PEAR_LOG_DEBUG); //Get source data $source_name = $feedItem->get_author()->name; if (!isset($source_name) || $source_name == null || $source_name == "") { $source_name = "unknown"; } $source = \Swiftriver\Core\ObjectModel\ObjectFactories\SourceFactory::CreateSourceFromIdentifier($source_name, $channel->trusted); $source->name = $source_name; $source->email = $feedItem->get_author()->email; $source->parent = $channel->id; $source->type = $channel->type; $source->subType = $channel->subType; //Extract all the relevant feedItem info $title = $feedItem->get_title(); $description = $feedItem->get_description(); $contentLink = $feedItem->get_permalink(); $date = $feedItem->get_date(); $rawLocation = $feedItem->get_item_tags("http://www.georss.org/georss", "point"); $long = 0; $lat = 0; $name = ""; if (is_array($rawLocation)) { $lat_lon_array = split(" ", $rawLocation[0]["data"]); $long = $lat_lon_array[1]; $lat = $lat_lon_array[0]; $location = new \Swiftriver\Core\ObjectModel\GisData($long, $lat, $name); } //Create a new Content item $item = \Swiftriver\Core\ObjectModel\ObjectFactories\ContentFactory::CreateContent($source); if (isset($location)) { $item->gisData = array($location); } //Fill the Content Item $item->text[] = new \Swiftriver\Core\ObjectModel\LanguageSpecificText(null, $title, array($description)); $item->link = $contentLink; $item->date = strtotime($date); //Add the item to the Content array $contentItems[] = $item; } $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [END: Parsing feed items]", \PEAR_LOG_DEBUG); $logger->log("Core::Modules::SiSPS::Parsers::FlickrParser::GetAndParse [Method finished]", \PEAR_LOG_DEBUG); //return the content array return $contentItems; }