public function errorAction($exception)
{
$ini = \Sooh\Base\Ini::getInstance();
$render = $ini->viewRenderType();
switch ($render) {
case "json":
header('Content-type: application/json');
break;
case 'jsonp':
break;
default:
$ini->viewRenderType('json');
header('Content-type: application/json');
break;
}
if (is_a($exception, '\\Sooh\\DB\\Acl\\ErrNeedsLogin')) {
\Sooh\Base\Log\Data::getInstance()->ret = 'needs login';
// }elseif(is_a($exception,'\Prj\ErrCode')){
// $this->_view->assign('code',$exception->getCode());
// $this->_view->assign('msg',$exception->getMessage());
// \Sooh\Base\Log\Data::getInstance()->ret = $exception->getMessage();
// }elseif(is_a($exception,'\Prj\RetCode')){
// $this->_view->assign('code',200);
// $this->_view->assign('msg',$exception->getMessage());
} else {
var_log($exception);
$this->_view->assign('code', $exception->getCode());
$this->_view->assign('msg', $exception->getMessage());
\Sooh\Base\Log\Data::getInstance()->ret = $exception->getMessage();
}
}
public function checkArchiveForThread($thread, $forumid)
{
global $logService;
$logService->log('TRACE', 'Check Archive for thread', ' ' . var_log($thread) . var_log($forumid), '');
$sth = $this->prepare("select status from qwp_threads where forumid=? and thread=?");
$sth->bindParam(1, $forumid, PDO::PARAM_INT);
$sth->bindParam(2, $thread, PDO::PARAM_INT);
$sth->execute();
$status = $sth->fetch(PDO::FETCH_ASSOC)['status'];
if ($status == 2) {
$this->restoreThread($thread, $forumid);
return true;
}
return false;
}
$keys = $nmDataService->getDisqusApp($forumname);
if ($keys) {
$logService->log('D4TRACE', 'LANDING found app ', var_log($forumname, '$forumname'), '');
$environment['public_key'] = $keys['public_key'];
$environment['secret_key'] = $keys['secret_key'];
} else {
//if invalid forumname, same as no forumname
$logService->log('D4TRACE', 'LANDING not found app ', var_log($forumname, '$forumname'), '');
$keys = $nmDataService->getLoginApp();
$environment['public_key'] = $keys['public_key'];
$environment['secret_key'] = $keys['secret_key'];
}
} else {
if ($aid) {
$keys = $nmDataService->getApp($aid);
$logService->log('D4TRACE', 'LANDING specified app ', var_log($aid, '$forumname'), '');
$environment['public_key'] = $keys['public_key'];
$environment['secret_key'] = $keys['secret_key'];
} else {
$keys = $nmDataService->getLoginApp();
$logService->log('D4TRACE', 'LANDING login app ', '', '');
$environment['public_key'] = $keys['public_key'];
$environment['secret_key'] = $keys['secret_key'];
}
}
$appid = $keys['appid'];
$login_keys = $nmDataService->getLoginApp();
$environment['login_public_key'] = $login_keys['public_key'];
$environment['login_secret_key'] = $login_keys['secret_key'];
//adjust the callback address with view and forumname:
if ($view > 0) {
function remapCategoryid($type, $oldid, $newid, $parent)
{
global $ds;
global $identity;
global $error;
global $logService;
/**
1. remap pov_thread_categories
2. remap pov_subscriptions
3. get all the children
4. foreach figure out new category id. Call itself.
**/
//$logService->log('DEBUG','entered recurse remap ',var_log($oldid,'$oldid').';'.var_log($newid,'$newid'),'');
if ($type != 'merge') {
$ds->updateCategoryid('change_categoryid', $oldid, $newid, $parent, $identity);
} else {
$ds->removeCategory($oldid, $identity, $newid);
}
// only done once at the top level for merges
// $logService->log('DEBUG','0.1 entered recurse remap ',var_log($oldid,'$oldid').';'.var_log($newid,'$newid'),'');
$ds->remapThreadCategories($oldid, $newid, $identity);
// $logService->log('DEBUG','0.2 entered recurse remap ',var_log($oldid,'$oldid').';'.var_log($newid,'$newid'),'');
$ds->remapSubscriptions($oldid, $newid, $identity);
// $logService->log('DEBUG','2 entered recurse remap ',var_log($oldid,'$oldid').';'.var_log($newid,'$newid'),'');
$nodes = $ds->loadCategories($oldid);
// $logService->log('DEBUG',' 3entered recurse remap ',var_log($nodes,'$nodes'),'');
foreach ($nodes as $node) {
$child_oldid = $node['id'];
$child_newid = str_replace($oldid, $newid, $child_oldid);
$logService->log('DEBUG', 'recurse remap ', var_log($child_oldid, '$child_oldid') . ';' . var_log($child_newid, '$child_newid'), '');
remapCategoryid('move', $child_oldid, $child_newid, $newid);
}
}
/**
* Pull changes for this post
*
* @return boolean success
*/
protected function pull_post_data()
{
if (!parent::pull_post_data()) {
return false;
}
// sync children NOW, so we can embed attachments
$content = $this->post->post_content;
foreach ($this->attachment_syncers as $syncer) {
$success = $syncer->pull($this->attachment_force);
if ($success) {
if (isset($syncer->post_meta['pmp_audio_shortcode'])) {
$content = $syncer->post_meta['pmp_audio_shortcode'] . "\n" . $content;
} elseif (isset($syncer->post_meta['pmp_image_url']) && !has_post_thumbnail($this->post->ID)) {
update_post_meta($this->post->ID, '_thumbnail_id', $syncer->post->ID);
}
}
}
// save changes to content
if ($content != $this->post->post_content) {
$id_or_error = wp_update_post(array('ID' => $this->post->ID, 'post_content' => $content), true);
if (is_wp_error($id_or_error)) {
var_log("pull_post_data ERROR for [{$this->doc->attributes->guid}] - {$id_or_error->get_error_message()}");
return false;
}
$this->post = get_post($id_or_error);
}
return true;
}
/**
Types of Disqus searches
1. Context: postid
2. Search: email,username,user,string
3. Scope: identity, thread
4. List of postids: commands, rule posts
Results are processes but not acted upon (only marked and get ruleid, watch, etc set)
Issues:
Separate searches for children, level and parents? If not - then we must preserve ids in the table. How do we know that ids need rewriting?
Answer: each row containst context information: one of [queue, level, child, parent], tab_id, view, identity. Each search deletes all the previous
rows first.
A: Create a table of postid/parentid to fetch old contexts from disqus qwp_post_index. Also should have a field to indicate if came alone or with the full thread.
If the postid is not present in qwp_post_index (with the context) then get parents but put a pseudo post as a sibling into level grid explaining the issue and offering to fetch the entire
old thread. This will also put the context into a tear-away tab if not already there. When the thread is loaded the context is rebuilt and the pseudo-post gone.
B: Need to have stale post indication. WHen the local post is present, calculate first if is stale or not before going to discus. Formula should be
age of the post devided by the interval since last update < 2. Also means adding updatedAt field.
Update: add last:nnn search
**/
function backgroundSearch($scope, $ct, $cv, $search, $tab_id)
{
//this is slow version of loadQueueItems. First it does same as loadQueue saving postids, then goes to disqus and gets the same posts.
//Then compares the ids, if same only updates text and generates change tick, if new = new posts tick for the specific tab
// $posts=loadPosts($scope,$ct,$cv,$search,$tab_id);
/**
1. Determine the type of request
a. search
b. all posts for identity
c. all posts for thread
d. ids (commands or rule)
e. context:level
f: context:child
**/
global $control;
global $access_token;
global $unDataService;
global $nmDataService;
global $global_username;
global $logService;
global $view;
global $queueType;
global $global_emailHash;
if ($queueType == "queue" && empty($search) && empty($ct) && empty($cv)) {
echo json_encode(array("success" => true, "total" => 0, "posts" => 0));
return;
}
if ($scope == 'watch' && !$control['watch'] || $scope == 'deleted' && !$control['deleted']) {
echo json_encode(array("success" => true, "total" => 0, "posts" => 0));
return;
}
if ($scope == 'watch' || $scope == 'deleted' || $scope == 'approved' || $scope == 'unconfirmed') {
echo json_encode(array("success" => true, "total" => 0, "posts" => 0));
return;
}
$forumid = $control['forumid'];
$ds = $nmDataService->getForumHubDataService($forumid);
$fdata = $nmDataService->getForumData($forumid);
$disqus_forumid = $fdata['external_id'];
if ($scope == 'all' && !$control['deleted']) {
$scope = 'working';
}
$context = 'queue';
if (strpos($queueType, ':') !== FALSE) {
$exp = explode(':', $queueType);
$queueType = $exp[0];
$context = $exp[1];
}
if ($context == 'queue') {
$type = 'queue';
} else {
$type = 'post';
}
if ($ct == "thread") {
$context = 'root';
$type = 'root';
}
$selfsearch = false;
if ($search == 'rself') {
$search = '';
$selfsearch = true;
}
$source = "type=" . $type . ";" . "scope=" . $scope . ";" . "constraint_type=" . $ct . ";" . "constraint_value=" . $cv . ";" . "tab_id=" . $tab_id . ";" . "search=" . $search;
$logService->log('DEBUG', '@@@backgroundSearch', 'source::' . $source, $global_username);
if ($context == 'queue' && $scope == 'all' && !empty($ct) && $ct == 'identity') {
//b.
$username = $ds->getUsernameByIdentity($cv, $forumid);
$search = "user:"******"success" => false, "msg" => "ERROR1: Exception:" + var_log($x)));
exit;
}
} else {
if ($type == 'root' && !empty($ct) && $ct == 'thread') {
//c.
$thread = $cv;
$search_to_show = "thread:" . $thread;
if ($selfsearch) {
$search = $source;
}
// $logService->log('DEBUG','###backgroundSearch','direct_source(search)::'.$search.'::$search_to_show::'.$search_to_show,$global_username);
/* $ispresent=$ds->getThreadTitle($thread,$forumid);
if($ispresent) {
echo json_encode(array(
"success" => true,
"total" => 0,
"posts" => 0
));
return;
}*/
$isload = $ds->getThreadSearch($thread, $forumid);
if (!$isload) {
$go = $ds->startServerSearch($tab_id, $scope, 'thread', 'all', $forumid, $search_to_show, $search);
// the search param has the primary source of the search that created the pseudo post
if ($go) {
try {
set_time_limit(0);
$ds->startThreadSearch($thread, $forumid);
$ds->updateThreadPseudoPosts($thread, $forumid);
getThread($ds, $fdata, $forumid, $access_token, $disqus_forumid, $thread, $go);
$ds->removeThreadPseudoPosts($thread, $forumid);
$ds->completeServerSearch($tab_id, $scope, 'thread', 'all', $forumid, $search_to_show);
$ds->completeThreadSearch($thread, $forumid);
set_time_limit(30);
} catch (Exception $x) {
set_time_limit(30);
$ds->completeServerSearch($tab_id, $scope, 'thread', 'all', $forumid, $search_to_show);
$logService->log('ERROR', 'backgroundSearch', fe($x), $forumid);
echo json_encode(array("success" => false, "msg" => "ERROR2: Exception:" + $x->getMessage()));
exit;
}
}
}
} else {
if ($context == 'queue' && $scope == 'commands') {
//d.1
//get list of postids
$posts = array();
$username = $ds->getUsernameByIdentity($cv, $forumid);
$search = "user:"******"success" => false, "msg" => "ERROR2: Exception:" + var_log($x)));
exit;
}
}
}
} else {
if ($context == 'queue' && $scope == 'rules' && !empty($ct) && $ct == 'rule') {
//d.2
//get list of postids
$posts = array();
$username = $ds->getUsernameByIdentity($cv, $forumid);
$search = "rules:" . $username;
// this is for nice display purpose only
if ($ct == 'rule') {
$posts = $ds->getRulePostids($cv, $forumid);
if ($posts) {
$go = $ds->startServerSearch($tab_id, $scope, 'rule', 'posts', $forumid, $search, $source);
try {
if ($go) {
set_time_limit(0);
foreach ((array) $posts as $post) {
$postid = $post['postid'];
getPost($ds, $fdata, $forumid, $access_token, $disqus_forumid, $postid, $go);
}
$ds->completeServerSearch($tab_id, $scope, 'rule', 'posts', $forumid, $search);
set_time_limit(30);
}
} catch (Exception $x) {
set_time_limit(30);
if ($go) {
$ds->completeServerSearch($tab_id, $scope, 'rule', 'posts', $forumid, $search);
}
$logService->log('ERROR', 'backgroundSearch', fe($x), $forumid);
echo json_encode(array("success" => false, "msg" => "ERROR4: Exception:" + var_log($x)));
exit;
}
}
}
} else {
if ($context == 'level' && !empty($ct) && $ct == 'post') {
//e - also need to get parent
/**
A.Parent processing:
1. Get all parent context postids from qwp_post_index. Find all the postids that are either missing from tab_posts or are stale.
Get them from disqus. If any were missing - send updateparent tick to UI.
2. If the index does not have a complete chain - get a context (in disqus terms) from discus. Send updateparent tick to UI.
B. Level processing:
1. Check if the post has context. If yes - get all the level postids from index, compare them to tab_posts and check if any are stale. For missing or stale get from disqus.
No ticks - the new and changed messages will be handled routinely by crawler processPosts.
2. If no context send a tick to put a pseudo post up inviting user to get the complete thread loaded if they want to see the full context.
**/
/**
Archiving refactoring 4/1/2015
if parent==-1 and thread.state==2 get the thread from archive (restoreThread())
else
walk index, if(thread.state==2) for each index with missing post get it from archive, only else go to Disqus for a copy
The queue search on background will also search archive and restore found posts, without their threads and will not change the state of the thread
**/
$username = $ds->getUsernameByIdentity($cv, $forumid);
$search = "post:" . $cv;
// this is for nice display purpose only
$postid = $cv;
$hasparent = false;
$go = $ds->startServerSearch($tab_id, $scope, 'post', 'context', $forumid, $search, $source);
if ($go) {
set_time_limit(0);
try {
$index_postids = $ds->getParentPostidsFromIndex($cv, $forumid);
//check if it is complete chain
$len = sizeof($index_postids);
$last_post = null;
if ($len >= 1) {
$last_post = $index_postids[$len - 1];
}
$now = time();
if ($last_post && $last_post['parentid'] == -1) {
//complete chain
if ($last_post['postid'] != $postid) {
$hasparent = true;
} else {
/** 4/1/2015 **/
$ds->restoreThreadIfArchived($last_post['thread'], $forumid);
}
//$logService->log('DEBUG','backgroundSearch','Parent chain (size='.$len.') is present locally in index for postid='.$cv,$forumid);
$local_postids = $ds->getPostsByMissingPostids($index_postids, $forumid);
$skip_ids = array();
foreach ((array) $local_postids as $lp) {
//stale calculations
$age = $now - $lp['createdat'];
$sinceupdate = $now - $lp['updatedat'];
if ($sinceupdate != 0 && $age / $sinceupdate > 2) {
//fresh enough
$skip_ids[] = $lp['postid'];
}
}
$sent = false;
foreach ($index_postids as $post) {
// $logService->log('DEBUG','backgroundSearch','Inside index_postids loop for postid='.$cv,$forumid);
$postid = $post['postid'];
if (in_array($post['postid'], $skip_ids)) {
continue;
}
$updatedat = $ds->getPostUpdatedat($forumid, $postid);
if ($updatedat > $now) {
//another thread updated the post meanwhile
continue;
}
// $logService->log('DEBUG','backgroundSearch','Calling getPost for postid='.$cv,$forumid);
/** first try archive 4/1/2015 **/
getPost($ds, $fdata, $forumid, $access_token, $disqus_forumid, $postid, $go);
$sent = true;
}
} else {
// get a complete disqus context for potid
$logService->log('DEBUG', 'backgroundSearch:level', 'Getting complete disqusus context for postid=' . $cv, $forumid);
getContext($ds, $fdata, $forumid, $access_token, $disqus_forumid, $postid, $go);
// $logService->log('DEBUG',' after backgroundSearch:level','Getting complete disqusus context for postid='.$cv,$global_username);
}
//B: Level processing
$postid = $cv;
//check if in context
// $logService->log('DEBUG','begin:level','',$global_username);
$has_context = $ds->checkHasContextonIndex($postid, $forumid);
if ($has_context) {
/**
3/8/2015
Thread archiving refactoring. If has context the thread coule be archived
**/
if ($hasparent) {
$index_postids = $ds->getLevelPostidsFromIndex($postid, $forumid);
// $logService->log('DEBUG',' getLevelPostidsFromIndex:level',var_log($local_postids,'$index_postids'),$global_username);
$now = time();
$local_postids = $ds->getPostsByMissingPostids($index_postids, $forumid);
//$local_postids=$ds->getPostidsFromTabPosts($global_emailHash,$view,$tab_id,'level');
// $logService->log('DEBUG',' aftergetPostidsFromTabPosts:level',var_log($local_postids,'$local_postids'),$global_username);
$skip_ids = array();
foreach ((array) $local_postids as $lp) {
//stale calculations
$age = $now - $lp['createdat'];
$sinceupdate = $now - $lp['updatedat'];
if ($sinceupdate) {
if ($age / $sinceupdate > 2) {
//fresh enough
$skip_ids[] = $lp['postid'];
}
}
}
foreach ($index_postids as $post) {
$postid = $post['postid'];
if (in_array($post['postid'], $skip_ids)) {
continue;
}
$updatedat = $ds->getPostUpdatedat($forumid, $postid);
if ($updatedat > $now) {
//another thread updated the post meanwhile
continue;
}
// $logService->log('DEBUG',' calling getPost:level',$postid,$global_username);
/** first try archive 4/1/2015 **/
getPost($ds, $fdata, $forumid, $access_token, $disqus_forumid, $postid, $go);
}
} else {
//just this one post if at the top level
getPost($ds, $fdata, $forumid, $access_token, $disqus_forumid, $postid, $go);
}
} else {
//insert a pseudopost
// $logService->log('DEBUG','before insert pseudopost:level','',$global_username);
$ds->insertPseudopost($tab_id, "level", $postid, $forumid, $source);
// $logService->log('DEBUG','after insert pseudopost:level','',$global_username);
}
$ds->completeServerSearch($tab_id, $scope, 'post', 'context', $forumid, $search);
set_time_limit(30);
// $logService->log('DEBUG','exit backgroundSearch:level','',$global_username);
} catch (Exception $x) {
set_time_limit(30);
$ds->completeServerSearch($tab_id, $scope, 'post', 'context', $forumid, $search);
$logService->log('ERROR', 'backgroundSearch', fe($x), $forumid);
echo json_encode(array("success" => false, "msg" => "ERROR3: Exception:" + var_log($x)));
exit;
}
}
} else {
if ($context == 'child' && !empty($ct) && $ct == 'post') {
//f
/**
C. Child processing: same as level except no pseudo post needed
**/
$username = $ds->getUsernameByIdentity($cv, $forumid);
$search = "post:" . $cv;
// this is for nice display purpose only
$postid = $cv;
$go = $ds->startServerSearch($tab_id, $scope, 'post', 'children', $forumid, $search, $source);
if ($go) {
set_time_limit(0);
try {
$has_context = $ds->checkHasContextonIndex($cv, $forumid);
$postid = $cv;
if ($has_context) {
$index_postids = $ds->getChildPostidsFromIndex($postid, $forumid);
$now = time();
$local_postids = $ds->getPostsByMissingPostids($index_postids, $forumid);
//$local_postids=$ds->getPostidsFromTabPosts($global_emailHash,$view,$tab_id,'child');
$skip_ids = array();
foreach ((array) $local_postids as $lp) {
//stale calculations
$age = $now - $lp['createdat'];
$sinceupdate = $now - $lp['updatedat'];
//$logService->log('DEBUG','backgroundSearch:child stale calc','$age='.$age.';$sinceupdate='.$sinceupdate,$global_username);
if ($sinceupdate) {
if ($age / $sinceupdate > 2) {
//fresh enough
$skip_ids[] = $lp['postid'];
}
}
}
foreach ($index_postids as $post) {
$postid = $post['postid'];
if (in_array($post['postid'], $skip_ids)) {
continue;
}
getPost($ds, $fdata, $forumid, $access_token, $disqus_forumid, $postid, $go);
}
}
$ds->completeServerSearch($tab_id, $scope, 'post', 'children', $forumid, $search);
set_time_limit(30);
// $logService->log('DEBUG','exit backgroundSearch:child','',$global_username);
} catch (Exception $x) {
set_time_limit(30);
$ds->completeServerSearch($tab_id, $scope, 'post', 'children', $forumid, $search);
$logService->log('ERROR', 'backgroundSearch:children', fe($x), $forumid);
echo json_encode(array("success" => false, "msg" => "ERROR3: Exception:" + var_log($x)));
exit;
}
}
} else {
if ($context == 'queue') {
//a.
$go = false;
try {
$go = $ds->startServerSearch($tab_id, $scope, $ct, $cv, $forumid, $search, $source);
if ($go) {
set_time_limit(0);
$ds->archiveSearch($forumid, $scope, $ct, $cv, $search, $context);
$posts = disqus_search($ds, $fdata, $forumid, $access_token, $disqus_forumid, $search, 300, $go);
// limit free form search to 300 posts
set_time_limit(30);
//
// $logService->log('DEBUG','backgroundSearch','search::'.$search,$forumid);
if (strpos($search, 'last:') !== FALSE || strpos($search, 'days:') !== FALSE) {
// init load
// $logService->log('DEBUG','backgroundSearch 2','search::'.$search,$forumid);
$tabs = $ds->loadUserForumTabs($control, $view);
//$logService->log('DEBUG','backgroundSearch 3','tabs::'.var_log($tabs),$forumid);
foreach ($tabs as $tab) {
$tab_id2 = $tab['tab_id'];
// $logService->log('DEBUG','Dispatch tick',' TabPosts backgroundSearch tab_id='.$tab_id,$forumid);
}
}
//$logService->log('DEBUG','Dispatch tick',' ServerSearchLog backgroundSearch ',$forumid);
$ds->completeServerSearch($tab_id, $scope, $ct, $cv, $forumid, $search);
}
} catch (Exception $x) {
if ($go) {
$ds->completeServerSearch($tab_id, $scope, $ct, $cv, $forumid, $search);
}
$logService->log('ERROR', 'backgroundSearch', fe($x), $forumid);
echo json_encode(array("success" => false, "msg" => "ERROR: Exception:" + var_log($x)));
exit;
}
}
}
}
}
}
}
}
echo json_encode(array("success" => true, "total" => 0, "posts" => 0));
}
/**
* Pull changes for this attachment
*
* @return boolean success
*/
protected function pull_post_data()
{
$data = array('ID' => $this->post->ID);
$data['post_title'] = $this->doc->attributes->title;
$data['post_date'] = date('Y-m-d H:i:s', strtotime($this->doc->attributes->published));
$data['post_date_gmt'] = gmdate('Y-m-d H:i:s', strtotime($this->doc->attributes->published));
if (isset($this->doc->attributes->description)) {
$data['post_excerpt'] = $this->doc->attributes->description;
} else {
$data['post_excerpt'] = '';
}
// handle errors
$id_or_error = wp_update_post($data, true);
if (is_wp_error($id_or_error)) {
var_log("pull_post_data ERROR for {$this->post->post_type}[{$this->doc->attributes->guid}] - {$id_or_error->get_error_message()}");
return false;
}
$this->post = get_post($id_or_error);
return true;
}
/**
* Helper for getting group links
*
* @param $guid the group guid
* @return a link object
*/
protected function get_group_link($guid)
{
if (!$this->doc) {
return null;
}
$fetch_doc = $this->doc->link(\Pmp\Sdk::FETCH_DOC);
if (empty($fetch_doc)) {
var_log('WOH: unable to get the fetch-doc link from this document');
return null;
}
return (object) array('href' => $fetch_doc->expand(array('guid' => $guid)));
}
/**
* Deletes all related file to an attachment.
*
* @param Base\Model $resource Currently processed resource.
* @param string $attachment Attachment file name.
* @param string $name Name of the file.
*
* @access private
* @static
*
* @return void
*/
private static function deleteThumbnails(Base\Model $resource, $attachment, $name)
{
$_storage_path = $resource->attachmentsStoragePath($attachment);
foreach (self::$attachments[$attachment]['thumbnails'] as $thumbnail) {
try {
if (file_exists($_storage_path . "{$thumbnail['size']}-{$name}")) {
Helpers\File::delete($_storage_path . "{$thumbnail['size']}-{$name}");
}
} catch (\Exception $e) {
var_log($e->getMessage());
}
}
}
function curl_loadLink($url, $default_date, $default_description, $default_language, $default_title, $default_site_name)
{
global $logService;
global $error;
global $ds;
global $root;
global $host;
if (empty($approved)) {
$approved = 0;
}
$parsed = parse_url($url);
$host = $parsed['host'];
if (!$root) {
$root = $host;
}
$session = curl_init($url);
// indicates that we want the response back rather than just returning a "TRUE" string
curl_setopt($session, CURLOPT_RETURNTRANSFER, true);
curl_setopt($session, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($session, CURLOPT_SSL_VERIFYPEER, false);
curl_setopt($session, CURLOPT_SSL_VERIFYHOST, false);
curl_setopt($session, CURLOPT_HEADER, false);
curl_setopt($session, CURLOPT_TIMEOUT, 60);
curl_setopt($session, CURLOPT_ENCODING, "");
//curl_setopt ($session, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36");
$header = array();
$header[] = 'Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5';
$header[] = 'Connection: keep-alive';
$header[] = 'Keep-Alive: 300';
/*$header[] = 'Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7';
$header[] = 'Accept-Language: en-us,en;q=0.5';
*/
//curl_setopt($session, CURLOPT_HTTPHEADER,$header)
// execute GET and get the session backs
$t = microtime(TRUE) * 1000000;
$logService->log('TRACE LINK URL', 'CURL GET START', $url, '');
$html = curl_exec($session);
$logService->log('TRACE LINK URL', 'CURL GET STOP(' . number_format(microtime(TRUE) * 1000000 - $t) . 'mks)', '', '');
curl_close($session);
if (!$html) {
$error = "FAILED TO LOAD document: " . $url;
return false;
}
if (stripos($url, 'novayagazeta') === FALSE && stripos($url, 'sotnik') === FALSE && stripos($url, 'ej.ru') === FALSE && stripos($url, 'newtimes') === FALSE) {
$html = @mb_convert_encoding($html, 'HTML-ENTITIES', 'utf-8');
}
$body = strip_html_tags($html);
/*$file = fopen("body.html", "w") or die("Unable to open file!");
fwrite($file, $body);
fclose($file);*/
$logService->log('TRACE ', 'Stripped html', $content, '');
libxml_use_internal_errors(true);
// Yeah if you are so worried about using @ with warnings
$dom = new DomDocument();
/*$st=mb_convert_encoding($html, 'HTML-ENTITIES', 'UTF-8');
if($dom->loadHTML($st)===FALSE){
$error="FAILED TO PARSE document: ".$url;
return false;
}*/
$encodingHint = '<meta http-equiv="Content-Type" content="text/html; charset=utf-8">';
if (stripos($html, 'Windows-1251') === FALSE && stripos($url, 'ej.ru') === FALSE) {
$html = $encodingHint . $html;
}
if ($dom->loadHTML($html) === FALSE) {
$error = "FAILED TO PARSE document: " . $url;
return false;
}
$dom->preserveWhiteSpace = false;
$updated_time = '';
$modified_time = '';
$image = '';
$title = '';
$link = '';
$site_name = $root;
$lang = '';
$description = '';
$twitter_image = '';
$i = 0;
$metas = $dom->getElementsByTagName('meta');
$countMetas = sizeof($metas);
$author = '';
$locale = '';
$sailthru_date = '';
$dcterms_date = '';
$basic_date = '';
$basic_publish_date = '';
//politico
$date = '';
$dcterms_created = '';
$dcterms_modified = '';
$noconvert = false;
$noconvertPublished = false;
$pdate = 0;
$basic_description = '';
$published_time = '';
foreach ($metas as $tag) {
// $logService->log('DEBUG','meta',var_log($tag),'crawler');
$property = $tag->getAttribute('property');
$name = $tag->getAttribute('name');
$content = $tag->getAttribute('content');
$http_equiv = $tag->getAttribute('http-equiv');
//$logService->log('TRACE LINK ','metas',var_log($property,'$property').';'.var_log($name,'$name').';'.var_log($content,'$content'),'blah');
if ($property === 'og:image') {
$image = $content;
// $logService->log('DEBUG','1 og:image=',$image,'crawler');
$parsed = parse_url($image);
if (!array_key_exists('host', $parsed)) {
$image = "http://" . $host . $image;
}
if (!is_available($image)) {
$image = "";
}
//$logService->log('DEBUG','og:image=',$image,'crawler');
continue;
}
if ($property === 'og:title') {
$title = $content;
$metafound = true;
continue;
}
if ($property === 'twitter:title') {
if (!$title) {
$title = $content;
}
$metafound = true;
continue;
}
if ($property === 'og:url') {
$link = $content;
$metafound = true;
$parsed = parse_url($link);
if (!array_key_exists('host', $parsed)) {
$link = "http://" . $host . $link;
}
continue;
}
if ($property === 'og:site_name') {
$site_name = $content;
continue;
}
if ($property === 'og:locale') {
$locale = $content;
//$logService->log('DEBUG','og:locale=',$locale,'crawler');
continue;
}
if ($property === 'og:description') {
$description = $content;
continue;
}
if ($property === 'twitter:description') {
if (!$description) {
$description = $content;
}
continue;
}
if ($property === 'og:date') {
//CNN, Thank you!
$date = $content;
continue;
}
if ($property === 'og:pubdate') {
//CNN, Thank you!
$date = $content;
continue;
}
if ($name === 'og:image') {
//espreso.tv
$image = $content;
$parsed = parse_url($image);
if (!array_key_exists('host', $parsed)) {
$image = "http://" . $host . $image;
}
if (!is_available($image)) {
$image = "";
}
continue;
}
if ($name === 'og:title') {
$title = $content;
$metafound = true;
continue;
}
if ($name === 'sailthru.title') {
if (!$title) {
$title = $content;
}
$metafound = true;
continue;
}
if ($name === 'og:url') {
$link = $content;
$metafound = true;
$parsed = parse_url($link);
if (!array_key_exists('host', $parsed)) {
$link = "http://" . $host . $link;
}
continue;
}
if ($name === 'og:site_name') {
$site_name = $content;
continue;
}
if ($name === 'og:locale') {
$locale = $content;
continue;
}
if (strcasecmp($name, 'language') == 0) {
$lang = $content;
continue;
}
if ($name === 'og:description') {
$description = $content;
continue;
}
if ($name === 'og:date') {
$date = $content;
continue;
}
if ($name === 'og:pubdate') {
$date = $content;
continue;
}
if ($name === 'twitter:image:src') {
$twiter_image = $content;
if (!is_available($twiter_image)) {
$twiter_image = "";
}
continue;
}
if ($name === 'sailthru.date') {
$sailthru_date = $content;
continue;
}
if ($name === 'sailthru.author') {
$author = $content;
continue;
}
if ($name === 'dcterms.date') {
$dcterms_date = $content;
continue;
}
if ($name === 'dcterms.created') {
$dcterms_created = $content;
continue;
}
if ($name === 'dcterms.modified') {
$dcterms_modified = $content;
continue;
}
if ($name === 'date') {
$basic_date = $content;
continue;
}
if ($name === 'publish_date') {
//Politico TODO If no time zone, add utc
$basic_publish_date = $content;
//.' utc';
continue;
}
if ($name === 'lastmod') {
$basic_date = $content;
continue;
}
if ($name === 'author') {
$author = $content;
continue;
}
if ($name === 'pdate') {
$year = substr($content, 0, 4);
$mo = substr($content, 4, 2);
$day = substr($content, 6, 2);
$pdate = $year . '-' . $mo . '-' . $day;
continue;
}
if ($name === 'dat') {
//NYTimes!
$basic_date = $dat;
//"May 12, 2015"
continue;
}
if (strcasecmp($name, 'description') == 0) {
//Ej, Mark Steyn - all caps - Sotnik
$basic_description = $content;
//
$metafound = true;
continue;
}
if ($property === 'article:published_time') {
$published_time = $content;
// $logService->log('DEBUG','Published time raw',$published_time,'crawler');
continue;
}
if ($property === 'article:modified_time') {
$updated_time = $content;
// $logService->log('DEBUG','Updated time raw',$updated_time,'crawler');
continue;
}
if ($http_equiv === 'Last-Modified') {
//censor.net.ua
$updated_time = $content;
}
if (strcasecmp($http_equiv, 'Content-Language') == 0) {
$lang = $content;
}
}
// $logService->log('DEBUG','inside-after metas',$url,'crawler');
// if($metafound){
/* $logService->log('cDEBUG','Meta found url='.$url,'Updated time'.$updated_time.';Published time='.$published_time
.';date='.$date.';title='.$title.';image='.$image.';description='.$description
,'','crawler');*/
/* if(!$title){
$logService->log('CDEBUG','NO TITLE url='.$url,'Updated time'.$updated_time.';Published time='.$published_time
.';date='.$date.';title='.$title.';image='.$image.';description='.$description
,'','crawler');
$titletags=$dom->getElementsByTagName('title'); // mark steyn
foreach ( $titletags as $titletag) {
$logService->log('CDEBUG','TITLE tag='.$titletag->textContent,'Updated time'.$updated_time.';Published time='.$published_time
.';date='.$date.';title='.$title.';image='.$image.';description='.$description
,'','crawler');
$title=$titletag->textContent;
}
}*/
if ($lang && !$locale) {
$locale = $lang . '_' . $lang;
}
//$logService->log('DEBUG','inside-after metas 001',$url,'crawler');
if (!$locale && !$lang) {
$locale = "en_US";
$lang = "en";
/* $re = "/(?:<html|(?<!^)\\G)\\h*(?:([^=\\n\\h]+)=(['\"]+)((?:\\\\\\2|(?!\\2).)*)\\2+)?/mi";
$logService->log('DEBUG','inside-after metas 0100',$url,'crawler');
ini_set("pcre.backtrack_limit",10000);
$logService->log('DEBUG','inside-after metas 0110',$url,'crawler');
preg_match_all($re,$html,$matches,PREG_SET_ORDER);
$logService->log('DEBUG','inside-after metas 0101',$url,'crawler');
foreach((array)$matches as $match){
if(sizeof($match)<3)
continue;
$t=$match[1];
if($t=='lang'||$t=='xml:lang'){
$lang=$match[3];
break;
}
}
$logService->log('DEBUG','inside-after metas 002',$url,'crawler');
$locale=str_replace('-', '_', $lang);
if(strlen($locale)==2)
$locale=$locale.'_'.strtoupper($locale);
//$lang=substr($lang,0,2);
*/
}
if (!$link) {
$link = $url;
}
$text = "";
// $logService->log('DEBUG','inside-after metas 0',$url,'crawler');
if (!$description) {
$description = $basic_description;
}
if (empty($title)) {
$title_tags = $dom->getElementsByTagName('title');
//ej.ru
if ($title_tags->length > 0) {
//$logService->log('DEBUG','Setting title tag',$title_tag->textContent,'crawler');
$title = $title_tags->item(0)->textContent;
}
}
if (stripos($title, 'steyn')) {
//yes, he is that important
$author = 'Mark Steyn';
}
if (stripos($title, 'Breitbart')) {
$author = 'Breitbart';
if (stripos($title, 'Author')) {
return false;
}
if (stripos($title, 'Stories')) {
return false;
}
}
if (stripos($title, 'derbyshire') || stripos($link, 'derbyshire')) {
//yes, he is that important
$author = 'John Derbyshire';
}
if (!$image) {
$image = $twitter_image;
}
// $logService->log('DEBUG','inside-after metas 2',$url,'crawler');
if (!$author) {
$title_tags = $dom->getElementsByTagName('title');
//ej.ru
if ($title_tags->length > 0) {
//$logService->log('DEBUG','Setting title tag',$title_tag->textContent,'crawler');
$t = $title_tags->item(0)->textContent;
if (stripos($t, 'The Weekly Standard')) {
$author = 'The Weekly Standard';
}
if (!$image) {
//sotnik tv
$image_tags = $dom->getElementsByTagName('img');
$found = false;
foreach ($image_tags as $img) {
$alt = $img->getAttribute('alt');
if (stripos($t, $alt) !== FALSE) {
$image = $img->getAttribute('src');
$found = true;
break;
}
}
if ($found) {
$parsed = parse_url($image);
if (!array_key_exists('host', $parsed)) {
$image = "http://" . $host . $image;
}
}
}
}
}
//if(/*!empty($description)&&*/!empty($title)){
// $logService->log('DEBUG','description and image',$url,'crawler');
$description = substr($description, 0, 1024);
if (!$updated_time) {
$updated_time = $date;
}
if (!$updated_time) {
$updated_time = $published_time;
}
if (!$updated_time) {
$updated_time = $dcterms_date;
}
if (!$published_time) {
$published_time = $dcterms_date;
}
if (!$updated_time) {
$updated_time = $basic_date;
}
if (!$published_time) {
$published_time = $basic_date;
}
if (!$updated_time) {
$updated_time = $basic_publish_date;
}
if (!$published_time) {
$published_time = $basic_publish_date;
}
if (!$updated_time) {
$updated_time = $dcterms_modified;
}
if (!$published_time) {
$published_time = $dcterms_modified;
}
if (!$updated_time) {
$updated_time = $pdate;
}
if (!$published_time && $pdate) {
$published_time = $pdate;
}
if (!$updated_time && !empty($sailthru_date)) {
$updated_time = $sailthru_date . " EDT";
}
//so far I've seen it in NRO only and being in East COast
if (!$published_time && !empty($sailthru_date)) {
$published_time = $sailthru_date . " EDT";
}
/*if(empty($updated_time)||empty($published_time)){
$logService->log('DEBUG','Empty time url='.$url,'Updated time'.$updated_time.';Published time='.$published_time
.';date='.$date.';dcterms_date='.$dcterms_date.';basic_publish_date='.$basic_publish_date.';basic_date='.$basic_date.';dcterms_modified='.$dcterms_modified.';pdate='.$pdate
,'','crawler');
}*/
if (!$updated_time && !empty($default_date)) {
$updated_time = $default_date;
}
if (!$published_time && !empty($default_date)) {
$published_time = $default_date;
}
if (!$description && !empty($default_description) && stripos($url, 'huff') === FALSE) {
// staying away from using RSS data as defaults on Huffington Post due to their posting
$description = $default_description;
}
if (!$title && !empty($default_title) && stripos($url, 'huff') === FALSE) {
$title = $default_title;
}
if (!$locale && !empty($default_language)) {
$locale = $default_language;
}
if (!$site_name && !empty($default_site_name)) {
$site_name = $default_site_name;
}
$published_time = strtotime($published_time);
$updated_time = strtotime($updated_time);
if (stripos($title, 'Mark Levin')) {
//yes, he is that important
$author = 'Mark Levin';
}
if (stripos($url, 'sotnik')) {
$author = 'SotnikTV';
}
/* if(empty($updated_time)||empty($published_time)){
$logService->log('DEBUG','2 Empty time url='.$url,'Updated time'.$updated_time.';Published time='.$published_time
.';date='.$date.';dcterms_date='.$dcterms_date.';basic_publish_date='.$basic_publish_date.';basic_date'.$basic_date.';dcterms_modified'.$dcterms_modified.';pdate'.$pdate
,'','crawler');
}*/
// $logService->log('DEBUG','Encoding='.$dom->encoding.';Updated time',$updated_time,'crawler');
/*if(!$locale)
$locale="en_US";*/
if (!$site_name) {
$site_name = $root;
}
// $logService->log('DEBUG','Encoding='. mb_detect_encoding($title).';Title',$title,'crawler');
if (stripos($description, '<') !== FALSE) {
$description = '';
}
if (stripos($html, '1251') !== FALSE) {
$title = mb_convert_encoding($title, "utf-8", "auto");
$logService->log('DEBUG', 'After conversion Encoding=' . $dom->encoding . ';Title', $title, 'crawler');
// $logService->log('DEBUG','ABefore conversion Encoding='.$dom->encoding.';Description',$description,'crawler');
$description = mb_convert_encoding($description, "utf-8", "auto");
$logService->log('DEBUG', 'After conversion Encoding=' . $dom->encoding . ';Description', $description, 'crawler');
if (!$locale) {
$locale = "ru_RU";
}
}
if (stripos($url, 'ej.ru')) {
$site_name = 'Ежедневный Журнал';
$locale = "ru_RU";
}
if (stripos($url, 'slon.ru')) {
$site_name = 'SLON';
$locale = "ru_RU";
$author = "Slon";
}
if (stripos($url, 'liga.net')) {
$locale = "ru_RU";
}
if (stripos($url, 'zn.ua')) {
$locale = "ru_RU";
}
if (stripos($url, 'liga.net/ua')) {
$locale = "uk_UA";
}
if (stripos($url, 'alternet')) {
continue;
}
if (stripos($url, 'dt.ua')) {
$locale = "uk_UA";
}
if (stripos($url, 'obozrevatel')) {
$locale = "ru_RU";
}
if (stripos($url, 'nv.ua')) {
$locale = "ru_RU";
}
if (stripos($url, 'gordonua.com')) {
$locale = "ru_RU";
}
if (stripos($url, 'tvrain')) {
$locale = "ru_RU";
}
if (stripos($url, 'english.gordonua')) {
$locale = "en_EN";
}
if (stripos($url, 'hromadske')) {
$locale = "uk_UA";
}
if (stripos($url, 'telegraph')) {
$locale = "en_EN";
}
if (stripos($url, 'politico') && !$locale) {
$locale = "en_EN";
}
if (stripos($url, 'americanthinker')) {
$site_name = 'American Thinker';
$locale = "en_EN";
}
if (stripos($url, 'washingtonpost')) {
$locale = "en_EN";
}
if (stripos($url, 'thenation')) {
$locale = "en_EN";
}
if (stripos($url, 'wsj')) {
$locale = "en_EN";
}
if (mb_stripos($url, 'cnn') && !$locale) {
$locale = "en_EN";
}
if (mb_stripos($url, 'salon') && !$locale) {
$locale = "en_EN";
}
if (mb_stripos($url, 'politico') && !$locale) {
$locale = "en_EN";
}
if (mb_stripos($url, 'petrimaz') && !$locale) {
$locale = "ru_RU";
}
/*if(stripos($url,'lesoir')&&!$locale){
$locale="fr_FR";
}*/
if (!$locale && (mb_stripos($url, '.be') || mb_stripos($url, '.fr'))) {
$locale = "fr_FR";
}
if (!$author && stripos($url, 'nationalreview')) {
$site_name = 'National Review';
$locale = "en_EN";
$author = "National Review";
if (stripos($title, 'author')) {
return false;
}
}
if ($locale) {
if (stripos($locale, "en") === 0) {
$locale = 'en_EN';
}
}
if ($locale) {
if (stripos($locale, "ru") === 0) {
$locale = 'ru_RU';
}
}
if ($locale) {
if (stripos($locale, "uk") === 0) {
$locale = 'uk_UA';
}
}
if ($locale) {
if (stripos($locale, "en") === 0) {
$locale = 'en_EN';
}
}
if ($author) {
/*$logService->log('CDEBUG','AUTHOR! url='.$url,'Updated time'.$updated_time.';Published time='.$published_time
.';date='.$date.';dcterms_date='.$dcterms_date.';basic_publish_date='.$basic_publish_date.';basic_date'.$basic_date.';dcterms_modified'.$dcterms_modified.';pdate'.$pdate
,'','crawler');*/
$author_record = $ds->checkAuthor($author);
/*$logService->log('CDEBUG','AUTHOR record='.var_log($author_record),'Updated time'.$updated_time.';Published time='.$published_time
.';date='.$date.';dcterms_date='.$dcterms_date.';basic_publish_date='.$basic_publish_date.';basic_date'.$basic_date.';dcterms_modified'.$dcterms_modified.';pdate'.$pdate
,'','crawler');*/
/*$logService->log('CDEBUG','before image='.$author_record['image'],'$image='.$image.' author='.$author.' author_record='.$author_record
.';date='.$date.';dcterms_date='.$dcterms_date.';basic_publish_date='.$basic_publish_date.';basic_date'.$basic_date.';dcterms_modified'.$dcterms_modified.';pdate'.$pdate
,'','crawler');*/
if ($author_record) {
/* $logService->log('CDEBUG','image='.$author_record['image'],'Updated time'.$updated_time.';Published time='.$published_time
.';date='.$date.';dcterms_date='.$dcterms_date.';basic_publish_date='.$basic_publish_date.';basic_date'.$basic_date.';dcterms_modified'.$dcterms_modified.';pdate'.$pdate
,'','crawler');*/
if ($author != "Breitbart") {
$image = $author_record['image'];
} else {
if (!$image) {
$image = $author_record['image'];
}
}
}
}
//last redoubt
if (!$description) {
$xpath = new DOMXPath($dom);
$textnodes = $xpath->query('//text()');
foreach ($textnodes as $node) {
$text = $node->textContent;
if (strlen($text) < 128) {
continue;
}
if (stripos($text, 'CDATA') !== FALSE) {
continue;
}
if (stripos($text, 'window.') !== FALSE) {
continue;
}
if (stripos($text, 'viewport') !== FALSE) {
continue;
}
if (stripos($text, 'jQuery') !== FALSE) {
continue;
}
if (stripos($text, 'function') !== FALSE) {
continue;
}
if (stripos($text, 'google') !== FALSE) {
continue;
}
if (stripos($text, 'google') !== FALSE) {
continue;
}
if (stripos($text, '!important') !== FALSE) {
continue;
}
if (stripos($text, 'font') !== FALSE) {
continue;
}
if (stripos($text, 'css') !== FALSE) {
continue;
}
if (stripos($text, '$(') !== FALSE) {
continue;
}
if (stripos($text, '==') !== FALSE) {
continue;
}
if (stripos($text, '{') !== FALSE) {
continue;
}
if (strcasecmp($text, $title) == 0) {
continue;
}
if (strlen($text) > 128) {
$description = $text;
break;
}
}
}
if (!$image) {
$xpath = new DOMXPath($dom);
$imagenodes = $xpath->query('//img[@width>300][@height>300]');
foreach ($imagenodes as $node) {
$src = $node->getAttribute('src');
$image = $src;
break;
}
}
if (!$image) {
$logService->log('CDEBUG', 'SAVING WITH NO IMAGE time=' . $updated_time . ';Published time=' . $published_time, 'link=' . $link . ';title=' . $title . ';image=' . $image . ';site_name=' . $site_name . ';description=' . $description . ';locale=' . $locale, 'crawler');
$image = "/build/css/logo2pale.png";
}
//}
$ret = array("link" => $link, "image" => $image, "title" => $title, "description" => $description, "site_name" => $site_name, "published_time" => $published_time, "updated_time" => $updated_time, "author" => $author, "locale" => $locale, "body" => $body);
$logService->log('CDEBUG', 'returning metas url=' . $url, var_log($ret, '$ret'), '');
if ($author) {
$ret["author_categoryid"] = $author_record['categoryid'];
}
$ret['tags'] = $dom->getElementsByTagName('a');
return $ret;
}
public function respondByDefenderUser($postid, $message)
{
global $nmDataService;
global $logService;
global $error_code;
global $error;
global $forum_status;
global $tds;
global $appid;
global $logthreadid;
$rtxid = 0;
$ttxid = 0;
if ($forum_status != 2) {
return;
}
$appid = 1;
$message = $message . ' #' . $logthreadid;
$username = '******';
$unds = $nmDataService->getUsernodeDataService(1);
//$user=$unds->getDisqusUserByPrimaryName($username);
$user = $unds->getDisqusAppUserByUsername($username, $appid);
$logService->log('DEBUG C', 'respondByDefenderUser', var_log($user, '$user'), 'userManager');
$token = $user['auth_token'];
$refresh_token = $user['refresh_token'];
$identity = $user['identity'];
if (isset($tds)) {
$rtxid = $tds->forumTasks_shortStartTx('RespondToPost', 'username='******';appid=' . $appid . ';identity=' . $identity);
//$logService->log('DEBUG','forumTasks_shortStartTx(RespondToPost) returned'.$rtxid,'','userManager');
}
if (!respodToPost($token, $postid, $message, true)) {
if (isset($tds)) {
$tds->abortTx($rtxid, 'username='******';appid=' . $appid . ';identity=' . $identity . ';error=' . $error_code);
}
$logService->log('DEBUG C', 'Failed first attempt to respond by defender user ', 'code=' . $error_code, 'userManager');
if (isset($tds)) {
$ttxid = $tds->forumTasks_shortStartTx('RefreshToken', 'username='******';appid=' . $appid . ';identity=' . $identity . ';refresh_token=' . $refresh_token);
}
$token = refreshToken($refresh_token);
// $unds->refreshToken($identity);
if (!$token) {
if (isset($tds)) {
$tds->abortTx($ttxid, 'username='******';appid=' . $appid . ';identity=' . $identity . ';refresh_token=' . $refresh_token . ';error=' . $error);
}
} else {
if (isset($tds)) {
$tds->completeTx($ttxid);
}
$unds->refreshAppUserToken($user['identity'], $appid);
if (isset($tds)) {
$rtxid = $tds->forumTasks_shortStartTx('RespondToPost', 'username='******';appid=' . $appid . ';identity=' . $identity);
}
if (!respodToPost($token, $postid, $message, true)) {
if (isset($tds)) {
$tds->abortTx($rtxid, 'username='******';appid=' . $appid . ';identity=' . $identity . ';error=' . $error_code);
}
$logService->log('ERROR', 'Failed to respondToPost', 'Failed second attemt by defender user,giving up code=' . $error_code, 'userManager');
} else {
if (isset($tds)) {
$tds->completeTx($rtxid);
}
}
}
} else {
//if(isset($txid))
$tds->completeTx($rtxid);
}
}
function fixThreads($ds, $fdata, $forumid, $token)
{
global $logService;
try {
//$logService->log('TRACE C','before fixing threads','','');
$disqus_forumid = $fdata['external_id'];
$forumname = $fdata['forumname'];
$forumurl = $fdata['url'];
$forumicon = $fdata['icon'];
$threads = $ds->getUnknownThreads($forumid);
foreach ((array) $threads as $thread) {
$details = fetchThreadDetails($token, $disqus_forumid, $thread['thread']);
if (is_object($details) == false) {
$logService->log('ERROR', 'FETCH_POSTS ', 'Error fetching thread data from Disqus, going to sleep', $forumid);
sleep(60);
continue;
}
$ds->addThread($forumid, $details->id, html_entity_decode($details->title), $details->link, $forumname, $forumurl, $forumicon);
}
// $logService->log('TRACE C','after fixing threads, before fixing roots 1','','');
/* $threads=$ds->getThreadsWithoutRoot($forumid);
$logService->log('TRACE C','after fixing threads, before fixing roots 2','','');
foreach((array)$threads as $thread){
$logService->log('TRACE C','after fixing threads, before fixing roots 3','','');
$details=fetchThreadDetails($token,$disqus_forumid,$thread['thread']);
$logService->log('TRACE C','after fixing threads, before fixing roots 4','','');
if(is_object($details)==false){
$logService->log('ERROR','FETCH_POSTS (duration:'.number_format((microtime(TRUE)*1000000-$t)).'mks)','Error fetching thread data from Disqus, going to sleep','fetch');
sleep(60);
continue;
}
$logService->log('TRACE C','after fixing threads, before fixing roots 5','','');
$ds->addThreadRootOnly($forumid,$details->id,html_entity_decode($details->title),$details->link,$forumname,$forumurl,$forumicon);
}*/
// $logService->log('TRACE C','after fixing threads','',$forumid);
} catch (Exception $x) {
$logService->log('Exception', 'fixThreads', var_log($x), $forumid);
}
}
public function submitTopicCategory($cat, $approved, $threadid, $identity, $caffe)
{
global $logService;
$date = 'null';
$approvedby = null;
if ($approved) {
$date = 'now()';
$approvedby = $identity;
}
if (!$caffe) {
$caffe = 0;
}
$sth = $this->prepare("SELECT xid from pov_categories where id=?");
$logService->log('cDEBUG', 'Looking for category id=' . $cat, var_log($threadid, '$threadid') . var_log($cat, '$cat'), 'crawler');
$sth->bindParam(1, $cat, PDO::PARAM_STR);
$sth->execute();
$category_xid = $sth->fetch(PDO::FETCH_ASSOC)['xid'];
if (!$category_xid) {
// stub category no need to map, just for author image
$logService->log('cDEBUG', 'FALSE as category not found', var_log($thread_xid, '$thread_xid') . var_log($cat, '$cat'), 'crawler');
return;
}
$sth = $this->prepare("SELECT xid from pov_threads where threadid=?");
$sth->bindParam(1, $threadid, PDO::PARAM_STR);
$sth->execute();
$thread_xid = $sth->fetch(PDO::FETCH_ASSOC)['xid'];
// $this->beginTransaction();
$sth = $this->prepare("SELECT xid,identity from pov_thread_categories tc where thread_xid=? and category_xid=?");
$sth->bindParam(1, $thread_xid, PDO::PARAM_INT);
$sth->bindParam(2, $category_xid, PDO::PARAM_INT);
$sth->execute();
$res = $sth->fetch(PDO::FETCH_ASSOC);
if ($res) {
$logService->log('cDEBUG', 'FALSE as thread_category already exists', var_log($thread_xid, '$thread_xid') . var_log($category_xid, '$category_xid'), 'crawler');
return false;
}
$sth = $this->prepare("SELECT xid from pov_threads_view where xid=? and category_xid=?");
$sth->bindParam(1, $thread_xid, PDO::PARAM_INT);
$sth->bindParam(2, $category_xid, PDO::PARAM_INT);
$sth->execute();
$res = $sth->fetch(PDO::FETCH_ASSOC);
if ($res) {
$logService->log('cDEBUG', 'FALSE as threads_view already exists', var_log($thread_xid, '$thread_xid') . var_log($category_xid, '$category_xid'), 'crawler');
return false;
}
//saving identity and user_name while we have it with the share to avoid any future need for joins with the user db
$sth = $this->prepare("SELECT user_name,profileurl from pov_users where identity=?");
$sth->bindParam(1, $identity, PDO::PARAM_INT);
$sth->execute();
$user = $sth->fetch(PDO::FETCH_ASSOC);
$shared_by_user_name = $user['user_name'];
$shared_by_profileurl = $user['profileurl'];
$sth = $this->prepare("INSERT INTO pov_thread_categories (threadid,categoryid,identity,approvedby,approval_date,approval_status,request_date,thread_xid,category_xid,user_name,shared_by_profileurl,caffe) values(?,?,?,?," . $date . ",?,now(),?,?,?,?,?)");
$sth->bindParam(1, $threadid, PDO::PARAM_STR);
$sth->bindParam(2, $cat, PDO::PARAM_STR);
$sth->bindParam(3, $identity, PDO::PARAM_STR);
$sth->bindParam(4, $approvedby, PDO::PARAM_STR);
$sth->bindParam(5, $approved, PDO::PARAM_INT);
$sth->bindParam(6, $thread_xid, PDO::PARAM_INT);
$sth->bindParam(7, $category_xid, PDO::PARAM_INT);
$sth->bindParam(8, $shared_by_user_name, PDO::PARAM_STR);
$sth->bindParam(9, $shared_by_profileurl, PDO::PARAM_STR);
$sth->bindParam(10, $caffe, PDO::PARAM_INT);
$sth->execute();
$xid = $this->lastInsertId();
if ($approved) {
// $logService->log('cDEBUG','Insert into threads_view',$xid,'crawler');
$sth = $this->prepare("INSERT INTO `pov_threads_view` (`xid`,`threadid`,`title`,`site_name`,`url`,`description`,`locale`,`identity`,`image`,`text`,`date`,`published_time`,`updated_time`,`shared_by_user_name`,`category_xid`,feed,reshare,shared_time,shared_by_identity,shared_by_profileurl) \n select `xid`,`threadid`,`title`,`site_name`,`url`,`description`,`locale`,`identity`,`image`,`text`,`date`,`published_time`,`updated_time`,? as shared_by_user_name,? as category_xid,feed,reshare, UNIX_TIMESTAMP(now()) as shared_time,? as shared_by_identity,? as shared_by_profileurl from pov_threads t where t.xid=?");
$sth->bindParam(1, $shared_by_user_name, PDO::PARAM_STR);
$sth->bindParam(2, $category_xid, PDO::PARAM_INT);
$sth->bindParam(3, $identity, PDO::PARAM_INT);
$sth->bindParam(4, $shared_by_profileurl, PDO::PARAM_INT);
$sth->bindParam(5, $thread_xid, PDO::PARAM_INT);
$sth->execute();
//$logService->log('cDEBUG','Insert into threads_view done','','crawler');
}
$table = 'pov_thread_categories';
$action = 1;
$sth = $this->prepare("INSERT into pov_xfer (xid,`table`,`action`,`date`) VALUES(?,?,?,now())");
$sth->bindParam(1, $xid, PDO::PARAM_INT);
$sth->bindParam(2, $table, PDO::PARAM_STR);
$sth->bindParam(3, $action, PDO::PARAM_INT);
$sth->execute();
// $this->commit();
return true;
}
/**
* Removes a key-value pair.
*
* @param string $key Cache key.
*
* @return boolean
*/
public function remove($key)
{
$file = self::storagePath() . self::generateName($key);
try {
return File::delete($file);
} catch (\Exception $e) {
var_log($e->getMessage());
return false;
}
}
';
$top_left_menu = '
<li id="timeline_menu" class="active" data-toggle="tab"><a href="#">' . $lang_array['timeline'] . '</a></li>
<li id="interests" data-toggle="tab"><a href="#">' . $lang_array['select'] . '</a></li>
<li id="share" data-toggle="tab"><a href="#">' . $lang_array['share'] . '</a></li>';
if ($publisher || $approver) {
$top_left_menu .= '
<li id="approve" data-toggle="tab"><a href="#">' . $lang_array['approve'] . ' <span id="approval_badge" class="badge">' . $approval_count . '</span></a></li>
';
}
}
$forum_dropdown = '
<h4><div class="dropdown" style="float:left">
<span class="text-info">' . $lang_array['select_community'] . '</span><i class="fa fa-users" style="margin:0 10px 0 10px;color:#A00000;"></i> <a class="dropdown-toggle" data-toggle="dropdown" href="#">' . $forum_details['name'] . '
<span class="caret"></span></a>
<ul id="forum_dropdown" class="dropdown-menu">
</ul>
</div></h4>';
$community_dropdown = '
<h4> <div class="dropdown" >
<i class="fa fa-users" style="margin-right:10px;" data-toggle="tooltip" title="' . $lang_array['select_community'] . '"></i>
<a class="dropdown-toggle" data-toggle="dropdown" href="#">' . $forum_details['name'] . '
<span class="caret"></span></a>
<ul id="community_dropdown" class="dropdown-menu">
</ul>
</div></h4>';
$logService->log('TRACE', 'Logged user landing page', var_log($dynamic_script, '$dynamic_script'), $identity);
static function test2($a, $b, $c = NULL, $d = NULL)
{
$a1 = new NDate($a);
$b1 = $d ? new NDate($b, $c, $d) : ($c ? new NDate($b, $c) : new NDate($b));
if ($a1->equals($b1)) {
return;
}
error_log("----- TEST FAILED ----- ");
var_log($a);
var_log($b);
if (isset($c)) {
var_log($c);
}
if (isset($d)) {
var_log($d);
}
var_log($a1->get(MYSQL_DATETIME));
var_log($b1->get(MYSQL_DATETIME));
error_log("----------------------- ");
}
function crawl($url, $default_date, $default_description, $default_language, $default_title, $default_site_name, $nocrawl)
{
global $dom;
global $input;
global $root;
global $host;
global $links;
global $ds;
global $categoryid;
global $logService;
global $level;
global $start;
global $error;
echo "<br>crawl " . $url;
if (stripos($url, "hill") !== false) {
$break = 1;
}
$metas = curl_loadLink($url, $default_date, $default_description, $default_language, $default_title, $default_site_name);
$logService->log('DEBUG D4CRAWLER', 'returned 1 with metas ', $url, 'crawler');
$text = '';
$identity = "00e98e251067b27107189bd7c8316ba2";
// Hudson Wilde
$approved = 1;
$success = true;
if ($metas) {
if (stripos($metas['title'], "Archive") === FALSE && stripos($metas['title'], "Access Denied") === FALSE && stripos($metas['title'], "Print") === FALSE && stripos($metas['title'], "Page not found") === FALSE && stripos($metas['title'], "Articles: Breitbart") === FALSE && stripos($metas['title'], "Articles - Breitbart") === FALSE && stripos($metas['title'], "Hearst Magazines") === FALSE) {
$logService->log('DEBUG D4CRAWLER', 'returned with metas ', $url, 'crawler');
$threadid = '';
try {
$threadid = $ds->submitTopic($metas['link'], $metas['title'], $metas['image'], $metas['site_name'], $metas['description'], $text, $metas['locale'], $identity, $metas['published_time'], $metas['updated_time'], $metas['author'], 1, 0, $url);
$logService->log('DEBUG', 'Returned from submit topic', var_log($metas, '$metas'), 'crawler');
$ds->saveBody($threadid, $metas['body'] . $metas['description'] . $metas['title'] . $metas['site_name']);
} catch (Exception $x) {
$logService->log('DEBUG', 'Exception', var_log($x), 'd4crawler');
echo var_dump($x);
}
$success = false;
if ($threadid) {
try {
$logService->log('DEBUG', 'SubmitCategory threadid=' . $threadid . ',categoryid=' . $categoryid, $url, 'crawler');
$success = $ds->submitTopicCategory($categoryid, $approved, $threadid, $identity, 0);
if ($metas['author'] && $success) {
sleep(1);
// to change the shared_time and avoid dups in views
// $logService->log('DEBUG','SubmitCategory for author',$author_record['categoryid'],'crawler');
$suc = $ds->submitTopicCategory($metas['author_categoryid'], $approved, $threadid, $identity, 0);
// $logService->log('DEBUG','SubmitCategory for author succes='.$suc,$author_record['categoryid'],'crawler');
}
} catch (Exception $x) {
echo var_dump($x);
$logService->log('DEBUG Exception', 'Submitting topic category', var_log($x), 'd4crawler');
}
}
} else {
$logService->log('DEBUG D4Crawler', 'Disallowd metas', var_log($metas, '$metas'), 'd4crawler');
}
} else {
$logService->log('DEBUG D4Crawler', 'No metas return', '', 'd4crawler');
echo $error;
}
}
$client->setRedirectUri('http://newsline.world/php/oauth2.php');
$client->addScope(Google_Service_Webmasters::WEBMASTERS);
$client->setApplicationName("Newsline World");
$service = new Google_Service_Webmasters($client);
$token_rec = $ds->getGoogleToken("auth");
$token = null;
if ($token_rec) {
$token = $token_rec['token'];
}
if ($token) {
$client->setAccessToken($token);
}
//$client->revokeToken();
if ($client->isAccessTokenExpired()) {
$refresh_token = $token_rec['refresh_token'];
$client->refreshToken($refresh_token);
$token = $client->getAccessToken();
$ds->setGoogleToken("auth", $token, null);
}
} catch (Exception $x) {
$logService->log('ERROR', 'Tolen Exception', var_log($x), 'sitemapper');
echo var_dump($x);
}
try {
$logService->log('DEBUG', 'just before Submitted sitemap', $sitemap, 'sitemapper');
$service->sitemaps->submit('newsline.world', $sitemap);
$logService->log('DEBUG', 'Submitted sitemap', $sitemap, 'sitemapper');
} catch (Exception $x) {
$logService->log('ERROR Sitemapper', 'Exception', var_log($x), 'sitemapper');
echo var_dump($x);
}
/**
* When the PMP notification hub sends an update, handle it
*
* @since 0.3
*/
function pmp_do_notification_callback()
{
global $wpdb;
pmp_debug('========== pmp_do_notification_callback ==========');
$body = file_get_contents('php://input');
$hash = hash_hmac('sha1', $body, PMP_NOTIFICATIONS_SECRET);
// get a COMPLETE mapping of known-PMP-guids to top-level WP-posts
$pmp_post_data = $wpdb->get_results("select post_id, meta_value, post_parent " . "from {$wpdb->posts} join {$wpdb->postmeta} on (ID = post_id) " . "where meta_key = 'pmp_guid'", ARRAY_A);
// map to the TOP LEVEL post (attachments map to their parent)
$pmp_guids = array();
foreach ($pmp_post_data as $row) {
if ($row['post_parent'] > 0) {
$pmp_guids[$row['meta_value']] = $row['post_parent'];
} else {
$pmp_guids[$row['meta_value']] = $row['post_id'];
}
}
// check hub signature
if ($_SERVER['HTTP_X_HUB_SIGNATURE'] !== "sha1={$hash}") {
var_log('INVALID PMP notifications HTTP_X_HUB_SIGNATURE');
var_log(" Expected: sha1={$hash}");
var_log(" Got: " . $_SERVER['HTTP_X_HUB_SIGNATURE']);
return;
}
// parse xml pubsubhubbub body
$xml = simplexml_load_string($body);
foreach ($xml->channel->item as $item) {
$item_json = json_decode(json_encode($item));
$item_guid = $item_json->guid;
// look for Posts tied to that guid
if (isset($pmp_guids[$item_guid])) {
$post = get_post($pmp_guids[$item_guid]);
if ($post) {
$syncer = PmpPost::fromPost($post);
$syncer->pull();
}
}
}
}
function var_log(&$varInput, $var_name = '', $reference = '', $method = '=', $sub = false)
{
static $output;
static $depth;
if ($sub == false) {
$output = '';
$depth = 0;
$reference = $var_name;
try {
$var = serialize($varInput);
$var = unserialize($var);
} catch (Exception $x) {
$var =& $varInput;
}
} else {
++$depth;
$var =& $varInput;
}
// constants
$nl = "\n";
$block = 'a_big_recursion_protection_block';
$c = $depth;
$indent = '';
while ($c-- > 0) {
$indent .= '| ';
}
// if this has been parsed before
if (is_array($var) && isset($var[$block])) {
$real =& $var[$block];
$name =& $var['name'];
$type = gettype($real);
$output .= $indent . $var_name . ' ' . $method . '& ' . ($type == 'array' ? 'Array' : get_class($real)) . ' ' . $name . $nl;
// havent parsed this before
} else {
// insert recursion blocker
$var = array($block => $var, 'name' => $reference);
$theVar =& $var[$block];
// print it out
$type = gettype($theVar);
switch ($type) {
case 'array':
$output .= $indent . $var_name . ' ' . $method . ' Array (' . $nl;
$keys = array_keys($theVar);
foreach ($keys as $name) {
$value =& $theVar[$name];
var_log($value, $name, $reference . '["' . $name . '"]', '=', true);
}
$output .= $indent . ')' . $nl;
break;
case 'object':
$output .= $indent . $var_name . ' = ' . get_class($theVar) . ' {' . $nl;
foreach ($theVar as $name => $value) {
var_log($value, $name, $reference . '->' . $name, '->', true);
}
$output .= $indent . '}' . $nl;
break;
case 'string':
$output .= $indent . $var_name . ' ' . $method . ' "' . $theVar . '"' . $nl;
break;
default:
$output .= $indent . $var_name . ' ' . $method . ' (' . $type . ') ' . $theVar . $nl;
break;
}
// $var=$var[$block];
}
--$depth;
if ($sub == false) {
return $output;
}
}
/**
* override function get_message to add information about the changed
* blockinstances
*
* @param type $user
* @return type
*/
public function get_message($user)
{
$message = get_string_from_language($user->lang, 'newwatchlistmessageview1', 'activity', $this->viewinfo->get('title'), display_name($this->usr, $user));
try {
foreach ($this->blocktitles as $blocktitle) {
$message .= "\n" . get_string_from_language($user->lang, 'blockinstancenotification', 'activity', $blocktitle);
}
} catch (Exception $exc) {
var_log(var_export($exc, true));
}
return $message;
}
function test_series_featured_asc()
{
$this->markTestIncomplete('This test has not been implemented yet.');
of_set_option('series_enabled', 1);
// Create a landing page that sets the order to 'featured, ASC'
$landing = $this->factory->post->create(array('post_type' => 'cftl-tax-landing', 'tax_input' => array('series' => $this->series->slug)));
update_post_meta($landing, 'has_order', 'featured, ASC');
// Post published before the current post in its series
$before_id = $this->factory->post->create(array('post_date' => '2013-01-01 00:00:00', 'tax_input' => array('series' => $this->series_id)));
// Post published after the current post in its series
$after_id = $this->factory->post->create(array('post_date' => '2015-01-01 00:00:00', 'tax_input' => array('series' => $this->series_id)));
// create a post that is featured in this taxonomy. It shall be the first.
$tf = $this->factory->term->create(array('taxonomy' => 'prominence'));
wp_update_term($tf, 'prominence', array('slug' => 'taxonomy-featured'));
$feat = $this->factory->post->create(array('series' => $this->series->slug, 'tax_input' => array('prominence' => 'taxonomy-featured')));
$lr = new Largo_Related(3, $this->considered);
$ids = $lr->ids();
var_log($feat);
var_log($ids);
$this->assertEquals(3, count($ids), "Largo_Related returned other than 3 posts");
$this->assertFalse($feat == $ids[2], "The featured post is the last post in the return.");
$this->assertEquals($feat, $ids[0], "The featured post is not the first post in the return.");
}
