protected function _fillTheXlsFile($xls, $firstSheet)
{
$row = $this->_form->getRow();
$this->_progressBar = new Zend_ProgressBar(new Kwf_Util_ProgressBar_Adapter_Cache($this->_getParam('progressNum')), 0, 100);
$reporter = new Reporter();
$reporter->exportFlightPlanToXls($xls, $firstSheet, $row, $this->_progressBar);
$this->_progressBar->finish();
}
public function run(Reporter $reporter)
{
$reporter->start();
foreach ($this->tests as $test) {
$test->run($reporter);
}
$reporter->end();
$reporter->summary();
}
public function run(Reporter $reporter)
{
$fn = $this->fn;
if (is_callable($fn)) {
try {
$fn();
$reporter->reportSucceededExample($this);
} catch (\Exception $ex) {
$reporter->reportFailedExample($this, $ex);
}
}
}
/**
* @param $title
* @param $message
* @param array $extra_vars
* @return string
*/
protected static function generateHTMLReport($title, $message, $extra_vars = array())
{
$helper = new Whoops\Util\TemplateHelper();
$templateFile = Reporter::getResource("views/layout.html.php");
$cssFile = Reporter::getResource("css/whoops.base.css");
$zeptoFile = Reporter::getResource("js/zepto.min.js");
$jsFile = Reporter::getResource("js/whoops.base.js");
$frames = array();
// List of variables that will be passed to the layout template.
$vars = array("page_title" => $title, "stylesheet" => file_get_contents($cssFile), "zepto" => file_get_contents($zeptoFile), "javascript" => file_get_contents($jsFile), "header" => Reporter::getResource("views/header.html.php"), "env_details" => Reporter::getResource("views/env_details.html.php"), "title" => $title, "name" => array($title), "message" => $message, "frames" => $frames, "has_frames" => false, "tables" => array());
if (count($extra_vars)) {
$vars['tables']['Passed debug data'] = $extra_vars;
}
$vars['tables']["Server/Request Data"] = $_SERVER;
$vars['tables']["GET Data"] = $_GET;
$vars['tables']["POST Data"] = $_POST;
$vars['tables']["Files"] = $_FILES;
$vars['tables']["Cookies"] = $_COOKIE;
$vars['tables']["Session"] = isset($_SESSION) ? $_SESSION : array();
$vars['tables']["Environment Variables"] = $_ENV;
$helper->setVariables($vars);
// Catch the rendered template and return him
// @todo let the render()-method return the rendered template
ob_start();
$helper->render($templateFile);
$template = ob_get_contents();
ob_end_clean();
return $template;
}
public function testReportVersion()
{
//create new instance
$instance = new Instance();
$instance->network_username = '******';
$instance->network = 'twitter';
$_SERVER['HTTP_HOST'] = "mytestthinkup";
//report version
$result = Reporter::reportVersion($instance);
$this->assertPattern("/http:\\/\\/thinkup.com\\/version.php\\?v\\=/", $result[0]);
$this->assertEqual($result[1], "http://mytestthinkup/?u=danica+mckellar&n=twitter");
$this->assertEqual($result[2], 200);
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('InstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('googleplus', true);
//get cached
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl Google+ users
$instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'google+');
if (isset($options['google_plus_client_id']->option_value) && isset($options['google_plus_client_secret']->option_value)) {
foreach ($instances as $instance) {
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$refresh_token = $tokens['oauth_access_token_secret'];
$instance_dao->updateLastRun($instance->id);
$google_plus_crawler = new GooglePlusCrawler($instance, $access_token);
$dashboard_module_cacher = new DashboardModuleCacher($instance);
try {
$google_plus_crawler->initializeInstanceUser($options['google_plus_client_id']->option_value, $options['google_plus_client_secret']->option_value, $access_token, $refresh_token, $current_owner->id);
$google_plus_crawler->fetchInstanceUserPosts();
} catch (Exception $e) {
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$dashboard_module_cacher->cacheDashboardModules();
$instance_dao->save($google_plus_crawler->instance, 0, $logger);
Reporter::reportVersion($instance);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('InstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('youtube', true);
//get cached
$max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20;
//convert to seconds
$max_crawl_time = $max_crawl_time * 60;
$developer_key = isset($options['developer_key']) ? $options['developer_key']->option_value : null;
$max_comments = isset($options['max_comments']) ? $options['max_comments']->option_value : null;
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl youtube users
$instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'youtube');
if (isset($options['youtube_client_id']->option_value) && isset($options['youtube_client_secret']->option_value)) {
foreach ($instances as $instance) {
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$refresh_token = $tokens['oauth_access_token_secret'];
$instance_dao->updateLastRun($instance->id);
$youtube_crawler = new YouTubeCrawler($instance, $access_token, $max_crawl_time, $developer_key, $max_comments);
$dashboard_module_cacher = new DashboardModuleCacher($instance);
try {
$youtube_crawler->initializeInstanceUser($options['youtube_client_id']->option_value, $options['youtube_client_secret']->option_value, $access_token, $refresh_token, $current_owner->id);
$youtube_crawler->fetchInstanceUserVideos();
} catch (Exception $e) {
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$dashboard_module_cacher->cacheDashboardModules();
$instance_dao->save($youtube_crawler->instance, 0, $logger);
Reporter::reportVersion($instance);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('InstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('foursquare', true);
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
$instances = $instance_dao->getAllActiveInstancesStalestFirstByNetwork('foursquare');
// Check the client id and secret are set or we can't crawl
if (isset($options['foursquare_client_id']->option_value) && isset($options['foursquare_client_secret']->option_value)) {
// For each instance of foursquare on this install
foreach ($instances as $instance) {
if (!$owner_instance_dao->doesOwnerHaveAccessToInstance($current_owner, $instance)) {
// Owner doesn't have access to this instance; let's not crawl it.
continue;
}
// Set the user name in the log
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
// Write to the log that we have started to collect data
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
// Get the OAuth tokens for this user
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
// Set the access token
$access_token = $tokens['oauth_access_token'];
// Update the last time we crawled
$instance_dao->updateLastRun($instance->id);
// Create a new crawler
$crawler = new FoursquareCrawler($instance, $access_token);
// Check the OAuth tokens we have are valid
try {
$logger->logInfo("About to initializeInstanceUser", __METHOD__ . ',' . __LINE__);
$user = $crawler->initializeInstanceUser($access_token, $current_owner->id);
if (isset($user) && $user instanceof User) {
$logger->logInfo("User initialized", __METHOD__ . ',' . __LINE__);
}
// Get the data we want and store it in the database
$logger->logInfo("About to fetchInstanceUserCheckins", __METHOD__ . ',' . __LINE__);
$crawler->fetchInstanceUserCheckins();
} catch (Exception $e) {
// Catch any errors that happen when we check the validity of the OAuth tokens
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$logger->logInfo("About to cache dashboard modules", __METHOD__ . ',' . __LINE__);
// Cache the insights to improve the page load speed of the dashboard
$dashboard_module_cacher = new DashboardModuleCacher($instance);
$dashboard_module_cacher->cacheDashboardModules();
$instance_dao->save($crawler->instance, 0, $logger);
Reporter::reportVersion($instance);
// Tell the user crawling was sucessful
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
}
/**
*
* @see Reporter::currentlyProcessing add the previous element to the tree and start a new element
* for the new file
*
* @param $phpFile the
* file currently processed
*/
public function currentlyProcessing($phpFile)
{
parent::currentlyProcessing($phpFile);
$this->endCurrentElement();
$this->startNewElement($phpFile);
}
public static function main()
{
$reporter = new Reporter();
$data = "Crystal Report Data";
$crystalReportMaker = new CrystalReportMaker($data);
$reporter->setReportMaker($crystalReportMaker);
$reporter->constructReport();
$report = $reporter->getReport();
$report->printReport();
echo "<br/>";
$data = "Html Data";
$htmlReportMaker = new HtmlReportMaker($data);
$reporter->setReportMaker($htmlReportMaker);
$reporter->constructReport();
$report = $reporter->getReport();
$report->printReport();
echo "<br/>";
$data = "PDF Data";
$pdfReportMaker = new PdfReportMaker($data);
$reporter->setReportMaker($pdfReportMaker);
$reporter->constructReport();
$report = $reporter->getReport();
$report->printReport();
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('InstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('facebook', true);
//get cached
$max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20;
//convert to seconds
$max_crawl_time = $max_crawl_time * 60;
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl Facebook user profiles and pages
$profiles = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook');
$pages = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook page');
$instances = array_merge($profiles, $pages);
foreach ($instances as $instance) {
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$instance_dao->updateLastRun($instance->id);
$facebook_crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time);
$dashboard_module_cacher = new DashboardModuleCacher($instance);
try {
$facebook_crawler->fetchPostsAndReplies();
} catch (APIOAuthException $e) {
//The access token is invalid, save in owner_instances table
$owner_instance_dao->setAuthError($current_owner->id, $instance->id, $e->getMessage());
//Send email alert
$this->sendInvalidOAuthEmailAlert($current_owner->email, $instance->network_username);
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
} catch (Exception $e) {
$logger->logUserError('EXCEPTION: ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$dashboard_module_cacher->cacheDashboardModules();
$instance_dao->save($facebook_crawler->instance, 0, $logger);
Reporter::reportVersion($instance);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
public function reportdetailAction()
{
if (empty($this->view->app_credits_report)) {
die;
}
$array_of_params = $this->_request->getParams();
if (!isset($array_of_params['fromdate']) || $array_of_params['fromdate'] == '' || !isset($array_of_params['todate']) || $array_of_params['todate'] == '') {
$array_of_params['fromdate'] = date('Y-m-d', strtotime("now -1 day"));
$array_of_params['todate'] = date('Y-m-d', strtotime("now -1 day"));
}
$fromdate = $array_of_params['fromdate'] . ' 00:00:00';
$todate = $array_of_params['todate'] . ' 23:59:59';
$this->view->param = $array_of_params;
$appIDs = $this->view->uApps;
$reporter = new Reporter();
$result = array();
$keys = array_keys($appIDs);
foreach ($keys as $appID) {
if ($appID == 'zing' || $appID == 'admin') {
continue;
}
$subresult = array();
$subresult = $reporter->summaryDaily($appID, $fromdate, $todate);
foreach ($subresult as $appByDate) {
$result['' . $appByDate['analyticDate']]['' . $appByDate['appID']] = $appByDate['amount'];
}
}
$subresult = $reporter->summaryDaily('zing', $fromdate, $todate);
foreach ($subresult as $appByDate) {
$result['' . $appByDate['analyticDate']]['zing'] = $appByDate['amount'];
}
$prevDate = date('Y-m-d', strtotime($fromdate . " -1 day"));
$subresult = $reporter->getRemainBalance($prevDate . " 00:00:00", $todate);
foreach ($subresult as $appByDate) {
$result['' . $appByDate['analyticDate']]['remainbalance'] = $appByDate['remainBalance'];
}
$this->view->data = $result;
if ($array_of_params['export'] == 'csv') {
$this->exportDetailCSVReport();
die;
} else {
$this->view->frmSearch = $this->view->render('index/app_report_detail.phtml');
$this->view->container = $this->view->render('index/index.phtml');
}
}
/**
* Render a child class view
*
* @param object $class
* @param string $options
* @return void
*/
public function renderChild($class, $options = null)
{
Reporter::newInstance()->render($class, $options);
}
protected function _fillTheXlsFile($xls, $firstSheet)
{
$row = $this->_form->getRow();
$this->_progressBar = new Zend_ProgressBar(new Kwf_Util_ProgressBar_Adapter_Cache($this->_getParam('progressNum')), 0, 100);
$reporter = new Reporter();
$xls = PHPExcel_IOFactory::load("./templates/flight_task_template.xls");
#$xls->setActiveSheetIndex();
$xls->setActiveSheetIndex(0);
$firstSheet = $xls->getActiveSheet();
$sheetId = 0;
$firstSheet = $xls->createSheet($sheetId);
$reporter->exportFlightTaskToXls($xls, $firstSheet, $row, $this->_progressBar);
$this->_progressBar->finish();
return $xls;
}
public function tranxbyappsummarysubmitAction()
{
$array_of_params = $this->_request->getParams();
$appserver = new AppInfo();
$allApps = $appserver->getAllAppInfo();
$allApps = $this->sortAppByID($allApps);
$this->view->allApps = $allApps;
if (!isset($array_of_params['fromdate']) || $array_of_params['fromdate'] == '' || !isset($array_of_params['todate']) || $array_of_params['todate'] == '') {
$array_of_params['fromdate'] = date('Y-m-d', strtotime("now -1 day"));
$array_of_params['todate'] = date('Y-m-d', strtotime("now -1 day"));
$array_of_params['radiogroup'] = 1;
}
$this->view->param = $array_of_params;
$fromdate = $array_of_params['fromdate'] . ' 00:00:00';
$todate = $array_of_params['todate'] . ' 23:59:59';
$appIDs = array();
$appserver = new AppInfo();
if ($array_of_params['appid'] == '0' || !isset($array_of_params['appid'])) {
$allApps = $appserver->getAllAppInfo();
$allApps = $this->sortAppByID($allApps);
foreach ($allApps as $app) {
$appIDs['' . $app->appID] = $app->appName;
}
} else {
$appIDs[$array_of_params['appid'] . ''] = $appserver->getAppName($array_of_params['appid']);
}
$reporter = new Reporter();
$result = array();
$keys = array_keys($appIDs);
foreach ($keys as $appID) {
$subresult = array();
switch ($array_of_params['radiogroup']) {
case 1:
$subresult = $reporter->summaryDaily($appID, $fromdate, $todate);
break;
case 2:
$subresult = $reporter->summary($appID, $fromdate, $todate);
break;
}
if (count($subresult) > 0 && $subresult[0]['appID'] != null) {
$result[] = $subresult;
}
}
$this->view->data = $result;
$this->view->frmSearch = $this->view->render('index/app_tranxbyapp_summary.phtml');
$this->view->container = $this->view->render('index/index.phtml');
}
/**
* 处理master发过来的命令
* @param resource $socket
* @param int $flag
*/
public function dealCmd($channel, $length, $buffer)
{
// 主进程挂了，完蛋了
if ($length == 0) {
$this->event->delAll($this->channel);
$this->notice("!!!!!!!!!!!!!!!!!!!!Master has gone !!!!!!!!!!!!!!!!!");
$this->onMasterDead();
return false;
}
// master发过来的命令字只有一个字节
$cmd = Cmd::decodeForWorker($buffer);
// 判断是哪个命令字
switch ($cmd) {
// 测试
case Cmd::CMD_TEST:
$this->reportToMaster($cmd, 'test ok');
break;
// 获取该worker进程包含的文件
// 获取该worker进程包含的文件
case Cmd::CMD_REPORT_INCLUDE_FILE:
$files = get_included_files();
$this->reportToMaster($cmd, $files);
$this->onAlarm();
break;
// 命令该worker停止服务
// 命令该worker停止服务
case Cmd::CMD_STOP_SERVE:
$this->reportToMaster($cmd, 1);
$this->stopServe();
break;
// 命令重启服务
// 命令重启服务
case Cmd::CMD_RESTART:
$this->reportToMaster($cmd, 1);
$this->stopServe();
break;
// 命令关闭通信管道
// 命令关闭通信管道
case Cmd::CMD_CLOSE_CHANNEL:
break;
// 命令上报worker状态信息给master
// 命令上报worker状态信息给master
case Cmd::CMD_REPORT_STATUS_FOR_MASTER:
$this->reportToMaster($cmd, array_merge($this->statusInfo, array('memory' => memory_get_usage(true))));
break;
// 上报worker状态信息给FileMonitor
// 上报worker状态信息给FileMonitor
case Cmd::CMD_REPORT_WORKER_STATUS:
Reporter::reportWorkerStatus(array_merge($this->statusInfo, array('memory' => memory_get_usage(true), 'worker_name' => get_class($this))));
break;
case Cmd::CMD_PING:
$this->reportToMaster(Cmd::CMD_PONG, '');
$this->onAlarm();
break;
// 未知命令
// 未知命令
default:
$this->reportToMaster(Cmd::CMD_UNKNOW, 'CMD UNKONW!!');
}
}
/**
* 初始化一些定时任务
* @return void
*/
protected static function initTask()
{
// 任务初始化
Task::init();
// 测试环境定时获取worker包含的文件
if (PHPServerConfig::get('ENV') == 'dev') {
// 定时获取worker包含的文件
Task::add(self::$commonWaitTimeLong, array('PHPServer', 'sendCmdToAll'), array(Cmd::CMD_REPORT_INCLUDE_FILE));
// 定时检测终端是否关闭
Task::add(self::$commonWaitTimeLong, array('PHPServer', 'checkTty'));
} else {
// 定时发送alarm命令
Task::add(self::$commonWaitTimeLong, array('PHPServer', 'sendCmdToAll'), array(Cmd::CMD_PING));
}
// 如果不支持inotify则上报文件给FileMonitor进程来监控文件更新
Task::add(self::$checkFilesTimeLong, function () {
Reporter::reportIncludedFiles(PHPServer::getFilesToInotify());
});
// 检查worker内存占用情况
Task::add(self::$checkStatusTimeLong, array('PHPServer', 'checkWorkersMemory'));
// 检查心跳情况
Task::add(self::$checkStatusTimeLong, array('PHPServer', 'checkPingInfo'));
// 开发环境定时清理master输出
if (PHPServerConfig::get('ENV') == 'dev') {
Task::add(self::$commonWaitTimeLong, function () {
@ob_clean();
});
}
}
/**
*
* @see Reporter::start
*/
public function start()
{
parent::start();
}
public function crawl()
{
$config = Config::getInstance();
$logger = Logger::getInstance();
$instance_dao = DAOFactory::getDAO('TwitterInstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$instance_hashtag_dao = DAOFactory::getDAO('InstanceHashtagDAO');
// get oauth values
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('twitter', true);
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
$instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'twitter');
foreach ($instances as $instance) {
$logger->setUsername($instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$num_twitter_errors = isset($options['num_twitter_errors']) ? $options['num_twitter_errors']->option_value : null;
$dashboard_module_cacher = new DashboardModuleCacher($instance);
try {
if (isset($tokens['oauth_access_token']) && $tokens['oauth_access_token'] != '' && isset($tokens['oauth_access_token_secret']) && $tokens['oauth_access_token_secret'] != '') {
$archive_limit = isset($options['archive_limit']->option_value) ? $options['archive_limit']->option_value : 3200;
$api = new CrawlerTwitterAPIAccessorOAuth($tokens['oauth_access_token'], $tokens['oauth_access_token_secret'], $options['oauth_consumer_key']->option_value, $options['oauth_consumer_secret']->option_value, $archive_limit, $num_twitter_errors);
$twitter_crawler = new TwitterCrawler($instance, $api);
$instance_dao->updateLastRun($instance->id);
$twitter_crawler->fetchInstanceUserTweets();
$twitter_crawler->fetchInstanceUserMentions();
$twitter_crawler->fetchInstanceUserFriends();
$twitter_crawler->fetchInstanceUserFollowers();
$twitter_crawler->fetchInstanceUserGroups();
$twitter_crawler->fetchRetweetsOfInstanceUser();
$twitter_crawler->fetchInstanceUserFavorites();
$twitter_crawler->updateStaleGroupMemberships();
$twitter_crawler->fetchStrayRepliedToTweets();
$twitter_crawler->fetchUserFriendsByIDs();
$twitter_crawler->fetchUnloadedFriendDetails();
$twitter_crawler->fetchUnloadedFollowerDetails();
$twitter_crawler->cleanUpFollows();
$twitter_crawler->updateFriendsProfiles();
//Retrieve search results for saved keyword/hashtags
$instances_hashtags = $instance_hashtag_dao->getByInstance($instance->id);
foreach ($instances_hashtags as $instance_hashtag) {
$twitter_crawler->fetchInstanceHashtagTweets($instance_hashtag);
}
} else {
throw new Exception('Missing Twitter OAuth tokens.');
}
} catch (Exception $e) {
$logger->logUserError(get_class($e) . " while crawling " . $instance->network_username . " on Twitter: " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$dashboard_module_cacher->cacheDashboardModules();
// Save instance
if (isset($twitter_crawler->user)) {
$instance_dao->save($instance, $twitter_crawler->user->post_count, $logger);
}
Reporter::reportVersion($instance);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . " on Twitter.", __METHOD__ . ',' . __LINE__);
}
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('InstagramInstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('instagram', true);
//get cached
$max_api_calls = isset($options['max_api_calls']) ? $options['max_api_calls']->option_value : 2500;
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl instagram user profiles and pages
$instances = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'instagram');
foreach ($instances as $instance) {
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$instance_dao->updateLastRun($instance->id);
$dashboard_module_cacher = new DashboardModuleCacher($instance);
try {
/**
* 1. Fetch user info, media + its likes and comments.
* 2. Fetch user's likes.
* 3. Fetch user's friends, and update stale relationships.
* 4. Fetch user's followers, and update stale relationships.
* 5. Update stale friends' profiles.
*/
$instagram_crawler = new InstagramCrawler($instance, $access_token, $max_api_calls);
$instagram_crawler->fetchPostsAndReplies();
$instagram_crawler->fetchLikes();
$instagram_crawler->fetchFriends();
$instagram_crawler->fetchFollowers();
$instagram_crawler->updateStaleFriendsProfiles();
} catch (Instagram\Core\ApiAuthException $e) {
//The access token is invalid, save in owner_instances table
$owner_instance_dao->setAuthErrorByTokens($instance->id, $access_token, '', $e->getMessage());
//Send email alert
//Get owner by auth tokens first, then send to that person
$owner_email_to_notify = $owner_instance_dao->getOwnerEmailByInstanceTokens($instance->id, $access_token, '');
$email_attempt = $this->sendInvalidOAuthEmailAlert($owner_email_to_notify, $instance->network_username);
if ($email_attempt) {
$logger->logUserInfo('Sent reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__);
} else {
$logger->logInfo('Didn\'t send reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__);
}
$logger->logUserError(get_class($e) . ' ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
} catch (Exception $e) {
$logger->logUserError(get_class($e) . ' ' . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$dashboard_module_cacher->cacheDashboardModules();
$instance_dao->save($instagram_crawler->instance, 0, $logger);
Reporter::reportVersion($instance);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
/**
*
* @see Reporter::stop Add a new line with the new file name
*
* @param $phpFile the
* file currently processed
*/
public function currentlyProcessing($phpFile)
{
parent::currentlyProcessing($phpFile);
$this->_write("\nFile: " . $this->currentPhpFile . PHP_EOL);
}
public function crawl()
{
$logger = Logger::getInstance();
$config = Config::getInstance();
$instance_dao = DAOFactory::getDAO('FacebookInstanceDAO');
$owner_instance_dao = DAOFactory::getDAO('OwnerInstanceDAO');
$owner_dao = DAOFactory::getDAO('OwnerDAO');
$plugin_option_dao = DAOFactory::GetDAO('PluginOptionDAO');
$options = $plugin_option_dao->getOptionsHash('facebook', true);
//get cached
$max_crawl_time = isset($options['max_crawl_time']) ? $options['max_crawl_time']->option_value : 20;
//convert to seconds
$max_crawl_time = $max_crawl_time * 60;
$current_owner = $owner_dao->getByEmail(Session::getLoggedInUser());
//crawl Facebook user profiles and pages
$profiles = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook');
$pages = $instance_dao->getActiveInstancesStalestFirstForOwnerByNetworkNoAuthError($current_owner, 'facebook page');
$instances = array_merge($profiles, $pages);
foreach ($instances as $instance) {
$logger->setUsername(ucwords($instance->network) . ' | ' . $instance->network_username);
$logger->logUserSuccess("Starting to collect data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
$tokens = $owner_instance_dao->getOAuthTokens($instance->id);
$access_token = $tokens['oauth_access_token'];
$instance_dao->updateLastRun($instance->id);
$facebook_crawler = new FacebookCrawler($instance, $access_token, $max_crawl_time);
$dashboard_module_cacher = new DashboardModuleCacher($instance);
try {
$facebook_crawler->fetchPostsAndReplies();
} catch (APIOAuthException $e) {
$logger->logUserError(get_class($e) . ": " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
//Don't send reauth email if it's app-level API rate limting
//https://developers.facebook.com/docs/reference/ads-api/api-rate-limiting/#applimit
if (strpos($e->getMessage(), 'Application request limit reached') === false && strpos($e->getMessage(), 'Please retry your request later.') === false) {
//The access token is invalid, save in owner_instances table
$owner_instance_dao->setAuthErrorByTokens($instance->id, $access_token, '', $e->getMessage());
//Send email alert
//Get owner by auth tokens first, then send to that person
$owner_email_to_notify = $owner_instance_dao->getOwnerEmailByInstanceTokens($instance->id, $access_token, '');
$email_attempt = $this->sendInvalidOAuthEmailAlert($owner_email_to_notify, $instance->network_username);
if ($email_attempt) {
$logger->logUserInfo('Sent reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__);
} else {
$logger->logInfo('Didn\'t send reauth email to ' . $owner_email_to_notify, __METHOD__ . ',' . __LINE__);
}
} else {
$logger->logInfo('Facebook API returned an error: ' . $e->getMessage() . ' Do nothing now and try again later', __METHOD__ . ',' . __LINE__);
}
} catch (Exception $e) {
$logger->logUserError(get_class($e) . ": " . $e->getMessage(), __METHOD__ . ',' . __LINE__);
}
$dashboard_module_cacher->cacheDashboardModules();
$instance_dao->save($facebook_crawler->instance, 0, $logger);
Reporter::reportVersion($instance);
$logger->logUserSuccess("Finished collecting data for " . $instance->network_username . "'s " . ucwords($instance->network), __METHOD__ . ',' . __LINE__);
}
}
<?php
include '../include/application/Csw.php';
include 'Reporter.php';
// --- MAIN ---
$reporter = new Reporter();
$q = html_entity_decode($_REQUEST['query']);
$result = $reporter->run($q);
header("Content-type: application/json; charset=utf-8");
echo json_encode($result);
