'',
$folder
) . '/',
'PRIORITY' => explode(
'/',
$arDirProperties['PRIORITY']
),
'children' => $children
];
}
public static function getPriority(array $priority, $depth)
{
$res = ($depth > count($priority) - 1) ? $priority[ count($priority) - 1 ] : $priority[ $depth ];
return $res;
}
}
?>
<pre>
<? $sitemap = new siteMap($_SERVER['DOCUMENT_ROOT']);
$sitemap->build();
?>
</pre>
function runJob($job, $force = false)
{
$stopAfterJob = false;
$startTime = microtime(true);
$this->log($job->task . ', started ' . date('g:i a \\a\\t D M n, Y', $startTime));
if (!isset($_GET['test']) && !$force) {
if ($job->isRunning == 1) {
$this->db->log('Cron race condition: ' . $job->task . ' last start= ' . $job->lastStart);
echo 'Warning: Cron Race condition<br />';
return;
} else {
$this->db->update("cronJobs", "isRunning=1,failureNoticeSent=0", "id={$job->id}");
}
}
//echo 'Task: '.$job->task."\n";
switch ($job->task) {
default:
// do nothing
break;
case 'updateUserLevels':
require_once 'teamBackend.class.php';
$teamObj = new teamBackend($this->db);
$teamObj->updateUserLevels();
break;
case 'updateSiteChallenges':
require_once 'teamBackend.class.php';
$teamObj = new teamBackend($this->db);
$teamObj->updateSiteChallenges();
break;
case 'updateCachedPointsAndChallenges':
require_once 'teamBackend.class.php';
$teamObj = new teamBackend($this->db);
$teamObj->updateCachedPointsAndChallenges();
break;
case 'calcWeeklyLeaders':
require_once 'teamBackend.class.php';
$teamObj = new teamBackend($this->db);
$teamObj->calcWeeklyLeaders();
break;
case 'updateUserLevels':
require_once 'teamBackend.class.php';
$teamObj = new teamBackend($this->db);
$teamObj->updateUserLevels();
case 'updateTwitter':
// post top stories to twitter
if (USE_TWITTER) {
require_once 'twitter.class.php';
$twitterObj = new twitter_old($this->db);
$twitterObj->postFeaturedStories();
$twitterObj->postNextTopStory();
}
break;
case 'microAccountsSync':
// sync twitter service accounts for micro blog room - done daily
if (defined('ENABLE_MICRO')) {
require_once PATH_FACEBOOK . "/classes/micro.class.php";
$mObj = new micro();
$mObj->cleanRoom();
try {
$mObj->resetFriends(false);
} catch (Exception $e) {
$this->log('Failed running ' . $job->task . ', Error: ' . $e);
// reset this cron task manually because of twitter class trown exceptions
$execTime = microtime(true) - $startTime;
$this->db->update("cronJobs", "nextRun=date_sub(NOW(), INTERVAL (0-{$job->freqMinutes}) MINUTE),lastExecTime={$execTime},lastStart='" . date('Y-m-d H:i:s', $startTime) . "',isRunning=0", "id={$job->id}");
}
}
break;
case 'microBlog':
// post top stories to micro blog room
if (defined('ENABLE_MICRO')) {
require_once PATH_FACEBOOK . "/classes/micro.class.php";
$mObj = new micro();
try {
$mObj->updateRoom();
} catch (Exception $e) {
$this->log('Failed running ' . $job->task . ', Error: ' . $e);
// reset this cron task manually because of twitter class trown exceptions
$execTime = microtime(true) - $startTime;
$this->db->update("cronJobs", "nextRun=date_sub(NOW(), INTERVAL (0-{$job->freqMinutes}) MINUTE),lastExecTime={$execTime},lastStart='" . date('Y-m-d H:i:s', $startTime) . "',isRunning=0", "id={$job->id}");
}
}
break;
case 'updateCache':
// build cached content for cover page layout
require_once 'cache.class.php';
$cacheObj = new cache($this->db);
$cacheObj->update();
break;
case 'syncProperties':
// sync Cloud properties with NewsCloud services
require_once 'apiCloud.class.php';
$apiObj = new apiCloud($this->db, $this->apiKey);
$props = $apiObj->syncProperties($this->cloudid);
require_once 'systemStatus.class.php';
$ssObj = new systemStatus($this->db);
$ssObj->setProperties($props['items'][0]);
break;
case 'syncAnnouncements':
require_once 'apiCloud.class.php';
$apiObj = new apiCloud($this->db, $this->apiKey);
$resp = $apiObj->syncAnnouncements($this->cloudid);
if ($resp[result] !== false) {
$itemlist = $resp[items];
require_once 'systemStatus.class.php';
$ssObj = new systemStatus($this->db);
$ssObj->resetAnnouncements();
if (count($itemlist) > 0) {
foreach ($itemlist as $data) {
$ssObj->insertState('announcement', html_entity_decode($data['announce']));
}
}
}
break;
case 'syncNewswire':
/* deprecated
require_once ('apiCloud.class.php');
$apiObj=new apiCloud($this->db,$this->apiKey);
$resp=$apiObj->syncNewswire($this->cloudid,$this->timeStrToUnixModB($job->lastItemTime));
$itemlist=$resp[items];
echo 'count: '.count($itemlist).'<br />';
if (count($itemlist)>0) {
require_once('newswire.class.php');
$nwObj=new newswire($this->db);
$lastItemTime=date('Y-m-d H:i:s',(time()-(6*30*24*3600))); // set to six months earlier
foreach ($itemlist as $data) {
$wire=$nwObj->serialize($data[title],$data[caption],$data[blogtitle],$data[webpage],$data[date],$data[blogid]);
$id=$nwObj->add($wire);
if ($data[date]>$lastItemTime)
$lastItemTime=$data[date];
if ($id===false)
echo 'skip '.$data[title].'<br />';
else
echo 'adding '.$data[title].' id->'.$id.'<br />';
}
$this->db->update("cronJobs","lastItemTime='$lastItemTime'","id=$job->id");
}
*/
break;
case 'syncLog':
require_once 'apiCloud.class.php';
$apiObj = new apiCloud($this->db, $this->apiKey);
// request server ask for log
$resp = $apiObj->requestSyncLog($this->cloudid, URL_HOME, $this->timeStrToUnixModB($job->lastStart));
// get result of log sync
$logResult = $resp[items][0][log];
require_once PATH_CORE . "/classes/log.class.php";
$logObj = new log($this->db);
// process results from sync operation
$result = $logObj->receive($logResult);
echo $result;
break;
case 'syncContent':
// bring content from NewsCloud for this cloud to the remote site
require_once 'apiCloud.class.php';
$apiObj = new apiCloud($this->db, $this->apiKey);
$resp = $apiObj->syncContent($this->cloudid, $this->timeStrToUnixModB($job->lastItemTime));
$itemlist = $resp[items];
if (count($itemlist) > 0) {
require_once 'content.class.php';
$cObj = new content($this->db);
// to do : set this to actual time
$lastItemTime = date('Y-m-d H:i:s', time() - 6 * 30 * 24 * 3600);
// set to six months earlier
foreach ($itemlist as $data) {
echo 'Contentid' . $data[contentid] . '<br />';
// to do: before we can do this below, we need to be syncing ncuid when new users register
// to do: get userid from ncUid
// lookup userid in user table where ncuid=submitbyid
// if not found make it 0
// to do: if external story, then check for local userid and set here
$story = $cObj->serialize($data[contentid], $data[title], $data[description], '', $data[webpage], $data[permalink], $data[submitbyid], $data[submit_member], $data[userid], $data[date], $data[avgrank], 0, $data[imageid]);
$id = $cObj->add($story);
// update comments table with new contentids
$cObj->updateCommentsTable($data[contentid], $id);
if ($data[date] > $lastItemTime) {
$lastItemTime = $data[date];
}
echo 'story added' . $id . '<br><br/>';
}
$this->db->update("cronJobs", "lastItemTime='{$lastItemTime}'", "id={$job->id}");
}
break;
case 'syncComments':
// bring comments from stories in this cloud from NewsCloud over to remote site
require_once 'content.class.php';
$cObj = new content($this->db);
$idList = $cObj->fetchRecentStoryList(14, 99, true);
$this->db->log('syncComments - stories to check for.', PATH_SYNCLOGFILE);
$this->db->log($idList, PATH_SYNCLOGFILE);
if ($idList != '') {
require_once 'comments.class.php';
$commentsObj = new comments($this->db);
require_once 'apiCloud.class.php';
$apiObj = new apiCloud($this->db, $this->apiKey);
$result = $apiObj->syncComments($this->cloudid, $idList, $this->timeStrToUnixModB($job->lastItemTime));
$itemlist = $result[items];
$this->db->log($itemlist, PATH_SYNCLOGFILE);
// update comment thread for each story
if (count($itemlist) > 0) {
$lastItemTime = date('Y-m-d H:i:s', time() - 6 * 30 * 24 * 3600);
// set to six months earlier
foreach ($itemlist as $data) {
$temp = 'Bring over contentid' . $data[contentid] . ' Commentid' . $data[commentid] . '<br />';
// to do: if external story, then check for local userid and set here
$comment = $commentsObj->remoteSerialize($data);
if ($data[date] > $lastItemTime) {
$lastItemTime = $data[date];
}
$id = $commentsObj->add($comment);
echo $temp . '<br />';
var_dump($comment);
$this->db->log($temp, PATH_SYNCLOGFILE);
$this->db->log($comment, PATH_SYNCLOGFILE);
}
}
$this->db->update("cronJobs", "lastItemTime='{$lastItemTime}'", "id={$job->id}");
}
break;
case 'syncScores':
require_once 'content.class.php';
$cObj = new content($this->db);
$idList = $cObj->fetchRecentStoryList(14, 99, true);
if ($idList != '') {
require_once 'apiCloud.class.php';
$apiObj = new apiCloud($this->db, $this->apiKey);
$resp = $apiObj->syncScores($this->cloudid, $idList, $this->timeStrToUnixModB($job->lastStart));
$itemlist = $resp[items];
//var_dump($resp);
if (count($itemlist) > 0) {
// update the score for each story with new votes
foreach ($itemlist as $data) {
$this->db->update("Content", "score={$data['score']}", "contentid={$data['contentid']}");
$temp = 'Set score of contentid:' . $data[contentid] . 'to ' . $data[score];
echo $temp . '<br />';
$this->db->log($temp, PATH_SYNCLOGFILE);
}
}
}
break;
case 'syncResources':
require_once 'apiCloud.class.php';
$apiObj = new apiCloud($this->db, $this->apiKey);
require_once 'resources.class.php';
$resObj = new resources($this->db);
$result = $apiObj->syncResources($this->cloudid);
$resObj->sync(html_entity_decode($result[items][0][resources]));
break;
case 'updateSiteMap':
$currentHour = date('G');
// 0 - 24
$currentDayOfWeek = date('w');
// day of week 0-6
$currentDayOfMonth = date('j');
// day of month 1-31
require_once 'siteMap.class.php';
$smObj = new siteMap($this->db);
// do always - build the map for content from the last hour
$smObj->buildMap('hourly');
// only do this as midnight
if ($currentHour == 0) {
// map to all the content from the last day
$smObj->buildMap('daily');
}
// only do this at 3 am on first day of week
if ($currentDayOfWeek = 0 and $currentHour == 3) {
// map to all the content from the last week, etc.
$smObj->buildMap('weekly');
}
// only do this at 2 am on first day of month
if ($currentDayOfMonth == 1 and $currentHour == 2) {
$smObj->buildMap('monthly');
}
// call buildIndexMap after updating any individual child maps above
// just the time stamps from each individual map file are updated in the indexmap
// warning: if a individual map hasn't been built - the index map won't include a reference to it
$smObj->buildIndexMap();
break;
case 'fetchFeeds':
// import stories from feeds
require_once 'feed.class.php';
$feedObj = new feed($this->db);
$feedObj->fetchBookmarks();
$feedObj->fetchFeeds();
$feedObj->fetchImages();
if ($feedObj->newStoryLoaded) {
// update features
}
break;
case 'logHourlyStats':
require_once 'statistics.class.php';
$statsObj = new statistics($this->db);
$statsObj->logHourlyStats();
break;
case 'facebookMinifeed':
require_once PATH_FACEBOOK . "/classes/app.class.php";
$app = new app(NULL, true);
$facebook =& $app->loadFacebookLibrary();
require_once PATH_FACEBOOK . '/classes/miniFeeds.class.php';
$feedObj = new miniFeeds($this->db);
$feedObj->loadFacebook($facebook);
$feedObj->updateMiniFeeds();
break;
case 'facebookProfileBoxes':
require_once PATH_FACEBOOK . "/classes/app.class.php";
$app = new app(NULL, true);
$facebook =& $app->loadFacebookLibrary();
require_once PATH_FACEBOOK . '/classes/profileBoxes.class.php';
$proObj = new profileBoxes($this->db);
$proObj->loadFacebook($facebook);
$proObj->updateProfileBoxes();
break;
case 'facebookEmailEngine':
// tbd
break;
case 'facebookAllocations':
// check nightly facebook allocations
$ssObj = new systemStatus($this->db);
/* initialize the SMT Facebook appliation class, NO Facebook library */
require_once PATH_FACEBOOK . "/classes/app.class.php";
$app = new app(NULL, true);
$facebook =& $app->loadFacebookLibrary();
$npd = $facebook->api_client->admin_getAllocation('notifications_per_day');
$ssObj->setState('notifications_per_day', $npd);
$ssObj->setState('announcement_notifications_per_week', $facebook->api_client->admin_getAllocation('announcement_notifications_per_week'));
$ssObj->setState('requests_per_day', $facebook->api_client->admin_getAllocation('requests_per_day'));
$ssObj->setState('emails_per_day', $facebook->api_client->admin_getAllocation('emails_per_day'));
break;
case 'facebookSendNotifications':
require_once PATH_FACEBOOK . "/classes/app.class.php";
$app = new app(NULL, true);
$facebook =& $app->loadFacebookLibrary();
require_once PATH_FACEBOOK . "/classes/shareStories.class.php";
$ssObj = new shareStories($app);
$ssObj->processNotifications();
break;
case 'facebookSendPromos':
/* not needed for now
if (date('G')==0) {
require_once PATH_FACEBOOK."/classes/promos.class.php";
$promoObj=new promos($this->db);
$promoObj->send();
}
*/
break;
case 'insertNewResearchData':
require_once PATH_CORE . "/classes/researchRawSession.class.php";
require_once PATH_CORE . "/classes/researchRawExtLink.class.php";
require_once PATH_CORE . "/classes/researchSessionLength.class.php";
require_once PATH_CORE . "/classes/researchLogDump.class.php";
require_once PATH_CORE . "/classes/researchUserCollective.class.php";
$rawExtLinkTable = new RawExtLinkTable($this->db);
$rawExtLinkTable->insertNewestData();
$rawSessionTable = new RawSessionTable($this->db);
$rawSessionTable->insertNewestData();
$sessionLengthTable = new SessionLengthTable($this->db);
$sessionLengthTable->insertNewestData();
$logDumpTable = new LogDumpTable($this->db);
$logDumpTable->insertNewestData();
$userCollectiveTable = new UserCollectiveTable($this->db);
$userCollectiveTable->assimilateUsers();
$stopAfterJob = true;
break;
case 'autoFeature':
require_once PATH_CORE . '/classes/content.class.php';
$cObj = new content($this->db);
$cObj->autoFeature();
break;
case 'cleanup':
require_once 'cleanup.class.php';
$cleanObj = new cleanup($this->db, 'daily');
break;
// deprecated
// deprecated
case 'syncFeedList':
require_once 'apiCloud.class.php';
$apiObj = new apiCloud($this->db, $this->apiKey);
$result = $apiObj->syncFeedList($this->cloudid);
require_once 'feed.class.php';
$feedObj = new feed($this->db);
$feedObj->syncFeedList($result[items]);
break;
}
$execTime = microtime(true) - $startTime;
$this->log('...completed in ' . $execTime . ' seconds.');
$this->history[$this->cntJobs]['task'] = $job->task;
$this->history[$this->cntJobs]['time'] = $execTime;
$this->cntJobs += 1;
$this->db->update("cronJobs", "nextRun=date_sub(NOW(), INTERVAL (0-{$job->freqMinutes}) MINUTE),lastExecTime={$execTime},lastStart='" . date('Y-m-d H:i:s', $startTime) . "',isRunning=0", "id={$job->id}");
if ($stopAfterJob) {
exit;
}
}
<?php
function debug($data)
{
print_r($data);
echo "\n";
}
include_once 'class.mycurl.php';
include_once 'class.siteMap.php';
if (!empty($argv[1]) and in_array($argv[1], array('--help', '-h'))) {
echo <<<HELP
This script takes a provided domain and curls the page that is rendered.
From the provided html, domain based links that are not rel="nofollow" are parsed out and stored in a url keyed list.
Each link from the url keyed list is compared against the sitemap list and added if missing.
That link is then curled and the process begins again.
\t--help (-h)\t\tHelp information.
HELP;
} else {
if (empty($argv[1])) {
echo <<<INVALID
To few arguments were provided. Pass a URL or --help (-h).
INVALID;
} else {
if (!empty($argv[1])) {
$SiteMap = new siteMap();
$SiteMap->generate($argv[1]);
debug($SiteMap->sitemap);
}
}
}