/**
* @brief Process incoming array of messages.
*
* Process an incoming array of messages which were obtained via pickup, and
* import, update, delete as directed.
*
* The message types handled here are 'activity' (e.g. posts), 'mail' ,
* 'profile', 'location' and 'channel_sync'.
*
* @param array $arr
* 'pickup' structure returned from remote site
* @param string $sender_url
* the url specified by the sender in the initial communication.
* We will verify the sender and url in each returned message structure and
* also verify that all the messages returned match the site url that we are
* currently processing.
*
* @returns array
* suitable for logging remotely, enumerating the processing results of each message/recipient combination
* * [0] => \e string $channel_hash
* * [1] => \e string $delivery_status
* * [2] => \e string $address
*/
function zot_import($arr, $sender_url)
{
$data = json_decode($arr['body'], true);
if (!$data) {
logger('zot_import: empty body');
return array();
}
if (array_key_exists('iv', $data)) {
$data = json_decode(crypto_unencapsulate($data, get_config('system', 'prvkey')), true);
}
if (!$data['success']) {
if ($data['message']) {
logger('remote pickup failed: ' . $data['message']);
}
return false;
}
$incoming = $data['pickup'];
$return = array();
if (is_array($incoming)) {
foreach ($incoming as $i) {
if (!is_array($i)) {
logger('incoming is not an array');
continue;
}
$result = null;
if (array_key_exists('iv', $i['notify'])) {
$i['notify'] = json_decode(crypto_unencapsulate($i['notify'], get_config('system', 'prvkey')), true);
}
logger('zot_import: notify: ' . print_r($i['notify'], true), LOGGER_DATA);
$hub = zot_gethub($i['notify']['sender']);
if (!$hub || $hub['hubloc_url'] != $sender_url) {
logger('zot_import: potential forgery: wrong site for sender: ' . $sender_url . ' != ' . print_r($i['notify'], true));
continue;
}
$message_request = array_key_exists('message_id', $i['notify']) ? true : false;
if ($message_request) {
logger('processing message request');
}
$i['notify']['sender']['hash'] = make_xchan_hash($i['notify']['sender']['guid'], $i['notify']['sender']['guid_sig']);
$deliveries = null;
if (array_key_exists('message', $i) && array_key_exists('type', $i['message']) && $i['message']['type'] === 'rating') {
// rating messages are processed only by directory servers
logger('Rating received: ' . print_r($arr, true), LOGGER_DATA);
$result = process_rating_delivery($i['notify']['sender'], $i['message']);
continue;
}
if (array_key_exists('recipients', $i['notify']) && count($i['notify']['recipients'])) {
logger('specific recipients');
$recip_arr = array();
foreach ($i['notify']['recipients'] as $recip) {
if (is_array($recip)) {
$recip_arr[] = make_xchan_hash($recip['guid'], $recip['guid_sig']);
}
}
$r = false;
if ($recip_arr) {
stringify_array_elms($recip_arr);
$recips = implode(',', $recip_arr);
$r = q("select channel_hash as hash from channel where channel_hash in ( " . $recips . " ) \n\t\t\t\t\t\tand channel_removed = 0 ");
}
if (!$r) {
logger('recips: no recipients on this site');
continue;
}
// It's a specifically targetted post. If we were sent a public_scope hint (likely),
// get rid of it so that it doesn't get stored and cause trouble.
if ($i && is_array($i) && array_key_exists('message', $i) && is_array($i['message']) && $i['message']['type'] === 'activity' && array_key_exists('public_scope', $i['message'])) {
unset($i['message']['public_scope']);
}
$deliveries = $r;
// We found somebody on this site that's in the recipient list.
} else {
if ($i['message'] && array_key_exists('flags', $i['message']) && in_array('private', $i['message']['flags']) && $i['message']['type'] === 'activity') {
if (array_key_exists('public_scope', $i['message']) && $i['message']['public_scope'] === 'public') {
// This should not happen but until we can stop it...
logger('private message was delivered with no recipients.');
continue;
}
}
logger('public post');
// Public post. look for any site members who are or may be accepting posts from this sender
// and who are allowed to see them based on the sender's permissions
$deliveries = allowed_public_recips($i);
if ($i['message'] && array_key_exists('type', $i['message']) && $i['message']['type'] === 'location') {
$sys = get_sys_channel();
$deliveries = array(array('hash' => $sys['xchan_hash']));
}
// if the scope is anything but 'public' we're going to store it as private regardless
// of the private flag on the post.
if ($i['message'] && array_key_exists('public_scope', $i['message']) && $i['message']['public_scope'] !== 'public') {
if (!array_key_exists('flags', $i['message'])) {
$i['message']['flags'] = array();
}
if (!in_array('private', $i['message']['flags'])) {
$i['message']['flags'][] = 'private';
}
}
}
// Go through the hash array and remove duplicates. array_unique() won't do this because the array is more than one level.
$no_dups = array();
if ($deliveries) {
foreach ($deliveries as $d) {
if (!in_array($d['hash'], $no_dups)) {
$no_dups[] = $d['hash'];
}
}
if ($no_dups) {
$deliveries = array();
foreach ($no_dups as $n) {
$deliveries[] = array('hash' => $n);
}
}
}
if (!$deliveries) {
logger('zot_import: no deliveries on this site');
continue;
}
if ($i['message']) {
if ($i['message']['type'] === 'activity') {
$arr = get_item_elements($i['message']);
$v = validate_item_elements($i['message'], $arr);
if (!$v['success']) {
logger('Activity rejected: ' . $v['message'] . ' ' . print_r($i['message'], true));
continue;
}
logger('Activity received: ' . print_r($arr, true), LOGGER_DATA);
logger('Activity recipients: ' . print_r($deliveries, true), LOGGER_DATA);
$relay = array_key_exists('flags', $i['message']) && in_array('relay', $i['message']['flags']) ? true : false;
$result = process_delivery($i['notify']['sender'], $arr, $deliveries, $relay, false, $message_request);
} elseif ($i['message']['type'] === 'mail') {
$arr = get_mail_elements($i['message']);
logger('Mail received: ' . print_r($arr, true), LOGGER_DATA);
logger('Mail recipients: ' . print_r($deliveries, true), LOGGER_DATA);
$result = process_mail_delivery($i['notify']['sender'], $arr, $deliveries);
} elseif ($i['message']['type'] === 'profile') {
$arr = get_profile_elements($i['message']);
logger('Profile received: ' . print_r($arr, true), LOGGER_DATA);
logger('Profile recipients: ' . print_r($deliveries, true), LOGGER_DATA);
$result = process_profile_delivery($i['notify']['sender'], $arr, $deliveries);
} elseif ($i['message']['type'] === 'channel_sync') {
// $arr = get_channelsync_elements($i['message']);
$arr = $i['message'];
logger('Channel sync received: ' . print_r($arr, true), LOGGER_DATA);
logger('Channel sync recipients: ' . print_r($deliveries, true), LOGGER_DATA);
$result = process_channel_sync_delivery($i['notify']['sender'], $arr, $deliveries);
} elseif ($i['message']['type'] === 'location') {
$arr = $i['message'];
logger('Location message received: ' . print_r($arr, true), LOGGER_DATA);
logger('Location message recipients: ' . print_r($deliveries, true), LOGGER_DATA);
$result = process_location_delivery($i['notify']['sender'], $arr, $deliveries);
}
}
if ($result) {
$return = array_merge($return, $result);
}
}
}
return $return;
}
public static function run($argc, $argv)
{
logger('onepoll: start');
if ($argc > 1 && intval($argv[1])) {
$contact_id = intval($argv[1]);
}
if (!$contact_id) {
logger('onepoll: no contact');
return;
}
$d = datetime_convert();
$contacts = q("SELECT abook.*, xchan.*, account.*\n\t\t\tFROM abook LEFT JOIN account on abook_account = account_id left join xchan on xchan_hash = abook_xchan \n\t\t\twhere abook_id = %d\n\t\t\tand abook_pending = 0 and abook_archived = 0 and abook_blocked = 0 and abook_ignored = 0\n\t\t\tAND (( account_flags = %d ) OR ( account_flags = %d )) limit 1", intval($contact_id), intval(ACCOUNT_OK), intval(ACCOUNT_UNVERIFIED));
if (!$contacts) {
logger('onepoll: abook_id not found: ' . $contact_id);
return;
}
$contact = $contacts[0];
$t = $contact['abook_updated'];
$importer_uid = $contact['abook_channel'];
$r = q("SELECT * from channel left join xchan on channel_hash = xchan_hash where channel_id = %d limit 1", intval($importer_uid));
if (!$r) {
return;
}
$importer = $r[0];
logger("onepoll: poll: ({$contact['id']}) IMPORTER: {$importer['xchan_name']}, CONTACT: {$contact['xchan_name']}");
$last_update = $contact['abook_updated'] === $contact['abook_created'] || $contact['abook_updated'] <= NULL_DATE ? datetime_convert('UTC', 'UTC', 'now - 7 days') : datetime_convert('UTC', 'UTC', $contact['abook_updated'] . ' - 2 days');
if ($contact['xchan_network'] === 'rss') {
logger('onepoll: processing feed ' . $contact['xchan_name'], LOGGER_DEBUG);
handle_feed($importer['channel_id'], $contact_id, $contact['xchan_hash']);
q("update abook set abook_connected = '%s' where abook_id = %d", dbesc(datetime_convert()), intval($contact['abook_id']));
return;
}
if ($contact['xchan_network'] !== 'zot') {
return;
}
// update permissions
$x = zot_refresh($contact, $importer);
$responded = false;
$updated = datetime_convert();
$connected = datetime_convert();
if (!$x) {
// mark for death by not updating abook_connected, this is caught in include/poller.php
q("update abook set abook_updated = '%s' where abook_id = %d", dbesc($updated), intval($contact['abook_id']));
} else {
q("update abook set abook_updated = '%s', abook_connected = '%s' where abook_id = %d", dbesc($updated), dbesc($connected), intval($contact['abook_id']));
$responded = true;
}
if (!$responded) {
return;
}
if ($contact['xchan_connurl']) {
$fetch_feed = true;
$x = null;
// They haven't given us permission to see their stream
$can_view_stream = intval(get_abconfig($importer_uid, $contact['abook_xchan'], 'their_perms', 'view_stream'));
if (!$can_view_stream) {
$fetch_feed = false;
}
// we haven't given them permission to send us their stream
$can_send_stream = intval(get_abconfig($importer_uid, $contact['abook_xchan'], 'my_perms', 'send_stream'));
if (!$can_send_stream) {
$fetch_feed = false;
}
if ($fetch_feed) {
$feedurl = str_replace('/poco/', '/zotfeed/', $contact['xchan_connurl']);
$feedurl .= '?f=&mindate=' . urlencode($last_update);
$x = z_fetch_url($feedurl);
logger('feed_update: ' . print_r($x, true), LOGGER_DATA);
}
if ($x && $x['success']) {
$total = 0;
logger('onepoll: feed update ' . $contact['xchan_name'] . ' ' . $feedurl);
$j = json_decode($x['body'], true);
if ($j['success'] && $j['messages']) {
foreach ($j['messages'] as $message) {
$results = process_delivery(array('hash' => $contact['xchan_hash']), get_item_elements($message), array(array('hash' => $importer['xchan_hash'])), false);
logger('onepoll: feed_update: process_delivery: ' . print_r($results, true), LOGGER_DATA);
$total++;
}
logger("onepoll: {$total} messages processed");
}
}
}
// update the poco details for this connection
if ($contact['xchan_connurl']) {
$r = q("SELECT xlink_id from xlink \n\t\t\t\twhere xlink_xchan = '%s' and xlink_updated > %s - INTERVAL %s and xlink_static = 0 limit 1", intval($contact['xchan_hash']), db_utcnow(), db_quoteinterval('1 DAY'));
if (!$r) {
poco_load($contact['xchan_hash'], $contact['xchan_connurl']);
}
}
return;
}
function externals_run($argv, $argc)
{
cli_startup();
$a = get_app();
$total = 0;
$attempts = 0;
logger('externals: startup', LOGGER_DEBUG);
// pull in some public posts
while ($total == 0 && $attempts < 3) {
$arr = array('url' => '');
call_hooks('externals_url_select', $arr);
if ($arr['url']) {
$url = $arr['url'];
} else {
$randfunc = db_getfunc('RAND');
// fixme this query does not deal with directory realms.
$r = q("select site_url, site_pull from site where site_url != '%s' and site_flags != %d and site_type = %d and site_dead = 0 order by {$randfunc} limit 1", dbesc(z_root()), intval(DIRECTORY_MODE_STANDALONE), intval(SITE_TYPE_ZOT));
if ($r) {
$url = $r[0]['site_url'];
}
}
$blacklisted = false;
if (!check_siteallowed($url)) {
logger('blacklisted site: ' . $url);
$blacklisted = true;
}
$attempts++;
// make sure we can eventually break out if somebody blacklists all known sites
if ($blacklisted) {
if ($attempts > 20) {
break;
}
$attempts--;
continue;
}
if ($url) {
if ($r[0]['site_pull'] !== NULL_DATE) {
$mindate = urlencode(datetime_convert('', '', $r[0]['site_pull'] . ' - 1 day'));
} else {
$days = get_config('externals', 'since_days');
if ($days === false) {
$days = 15;
}
$mindate = urlencode(datetime_convert('', '', 'now - ' . intval($days) . ' days'));
}
$feedurl = $url . '/zotfeed?f=&mindate=' . $mindate;
logger('externals: pulling public content from ' . $feedurl, LOGGER_DEBUG);
$x = z_fetch_url($feedurl);
if ($x && $x['success']) {
q("update site set site_pull = '%s' where site_url = '%s'", dbesc(datetime_convert()), dbesc($url));
$j = json_decode($x['body'], true);
if ($j['success'] && $j['messages']) {
$sys = get_sys_channel();
foreach ($j['messages'] as $message) {
// on these posts, clear any route info.
$message['route'] = '';
$results = process_delivery(array('hash' => 'undefined'), get_item_elements($message), array(array('hash' => $sys['xchan_hash'])), false, true);
$total++;
}
logger('externals: import_public_posts: ' . $total . ' messages imported', LOGGER_DEBUG);
}
}
}
}
}
function externals_run($argv, $argc)
{
cli_startup();
$a = get_app();
$total = 0;
$attempts = 0;
logger('externals: startup', LOGGER_DEBUG);
// pull in some public posts
while ($total == 0 && $attempts < 3) {
$arr = array('url' => '');
call_hooks('externals_url_select', $arr);
if ($arr['url']) {
$url = $arr['url'];
} else {
$randfunc = db_getfunc('RAND');
$r = q("select site_url, site_pull from site where site_url != '%s' and site_flags != %d order by {$randfunc} limit 1", dbesc(z_root()), intval(DIRECTORY_MODE_STANDALONE));
if ($r) {
$url = $r[0]['site_url'];
}
}
// Note: blacklisted sites must be stored in the config as an array.
// No simple way to turn this into a personal config because we have no identity here.
// For that we probably need a variant of superblock.
$blacklisted = false;
$bl1 = get_config('system', 'blacklisted_sites');
if (is_array($bl1) && $bl1) {
foreach ($bl1 as $bl) {
if ($bl && strpos($url, $bl) !== false) {
$blacklisted = true;
break;
}
}
}
$attempts++;
// make sure we can eventually break out if somebody blacklists all known sites
if ($blacklisted) {
if ($attempts > 20) {
break;
}
$attempts--;
continue;
}
if ($url) {
if ($r[0]['site_pull'] !== NULL_DATE) {
$mindate = urlencode(datetime_convert('', '', $r[0]['site_pull'] . ' - 1 day'));
} else {
$days = get_config('externals', 'since_days');
if ($days === false) {
$days = 15;
}
$mindate = urlencode(datetime_convert('', '', 'now - ' . intval($days) . ' days'));
}
$feedurl = $url . '/zotfeed?f=&mindate=' . $mindate;
logger('externals: pulling public content from ' . $feedurl, LOGGER_DEBUG);
$x = z_fetch_url($feedurl);
if ($x && $x['success']) {
q("update site set site_pull = '%s' where site_url = '%s'", dbesc(datetime_convert()), dbesc($url));
$j = json_decode($x['body'], true);
if ($j['success'] && $j['messages']) {
$sys = get_sys_channel();
foreach ($j['messages'] as $message) {
// on these posts, clear any route info.
$message['route'] = '';
$results = process_delivery(array('hash' => 'undefined'), get_item_elements($message), array(array('hash' => $sys['xchan_hash'])), false, true);
$total++;
// $z = q("select id from item where mid = '%s' and uid = %d limit 1",
// dbesc($message['message_id']),
// intval($sys['channel_id'])
// );
$z = null;
if ($z) {
$flag_bits = ITEM_WALL | ITEM_ORIGIN | ITEM_UPLINK;
// preserve the source
$r = q("update item set source_xchan = owner_xchan where id = %d", intval($z[0]['id']));
$r = q("update item set item_flags = ( item_flags | %d ), owner_xchan = '%s' \n\t\t\t\t\t\t\t\twhere id = %d", intval($flag_bits), dbesc($sys['xchan_hash']), intval($z[0]['id']));
}
}
logger('externals: import_public_posts: ' . $total . ' messages imported', LOGGER_DEBUG);
}
}
}
}
}
function onepoll_run($argv, $argc)
{
cli_startup();
$a = get_app();
logger('onepoll: start');
$manual_id = 0;
$generation = 0;
$force = false;
$restart = false;
if ($argc > 1 && intval($argv[1])) {
$contact_id = intval($argv[1]);
}
if (!$contact_id) {
logger('onepoll: no contact');
return;
}
$d = datetime_convert();
$contacts = q("SELECT abook.*, xchan.*, account.*\n\t\tFROM abook LEFT JOIN account on abook_account = account_id left join xchan on xchan_hash = abook_xchan \n\t\twhere abook_id = %d\n\t\tAND (( abook_flags & %d ) OR ( abook_flags = %d ))\n\t\tAND NOT ( abook_flags & %d )\n\t\tAND (( account_flags = %d ) OR ( account_flags = %d )) limit 1", intval($contact_id), intval(ABOOK_FLAG_HIDDEN | ABOOK_FLAG_PENDING | ABOOK_FLAG_UNCONNECTED | ABOOK_FLAG_FEED), intval(0), intval(ABOOK_FLAG_ARCHIVED | ABOOK_FLAG_BLOCKED | ABOOK_FLAG_IGNORED), intval(ACCOUNT_OK), intval(ACCOUNT_UNVERIFIED));
if (!$contacts) {
logger('onepoll: abook_id not found: ' . $contact_id);
return;
}
$contact = $contacts[0];
$t = $contact['abook_updated'];
$importer_uid = $contact['abook_channel'];
$r = q("SELECT * from channel left join xchan on channel_hash = xchan_hash where channel_id = %d limit 1", intval($importer_uid));
if (!$r) {
return;
}
$importer = $r[0];
logger("onepoll: poll: ({$contact['id']}) IMPORTER: {$importer['xchan_name']}, CONTACT: {$contact['xchan_name']}");
$last_update = $contact['abook_updated'] === $contact['abook_created'] || $contact['abook_updated'] === NULL_DATE ? datetime_convert('UTC', 'UTC', 'now - 7 days') : datetime_convert('UTC', 'UTC', $contact['abook_updated'] . ' - 2 days');
if ($contact['xchan_network'] === 'rss') {
logger('onepoll: processing feed ' . $contact['xchan_name'], LOGGER_DEBUG);
handle_feed($importer['channel_id'], $contact_id, $contact['xchan_hash']);
q("update abook set abook_connected = '%s' where abook_id = %d limit 1", dbesc(datetime_convert()), intval($contact['abook_id']));
return;
}
if ($contact['xchan_network'] !== 'zot') {
return;
}
// update permissions
$x = zot_refresh($contact, $importer);
$responded = false;
$updated = datetime_convert();
if (!$x) {
// mark for death by not updating abook_connected, this is caught in include/poller.php
q("update abook set abook_updated = '%s' where abook_id = %d limit 1", dbesc($updated), intval($contact['abook_id']));
} else {
q("update abook set abook_updated = '%s', abook_connected = '%s' where abook_id = %d limit 1", dbesc($updated), dbesc($updated), intval($contact['abook_id']));
$responded = true;
}
if (!$responded) {
return;
}
if ($contact['xchan_connurl']) {
$fetch_feed = true;
$x = null;
if (!($contact['abook_their_perms'] & PERMS_R_STREAM)) {
$fetch_feed = false;
}
if ($fetch_feed) {
$feedurl = str_replace('/poco/', '/zotfeed/', $contact['xchan_connurl']);
$x = z_fetch_url($feedurl . '?f=&mindate=' . urlencode($last_update));
logger('feed_update: ' . print_r($x, true), LOGGER_DATA);
}
if ($x && $x['success']) {
$total = 0;
logger('onepoll: feed update ' . $contact['xchan_name']);
$j = json_decode($x['body'], true);
if ($j['success'] && $j['messages']) {
foreach ($j['messages'] as $message) {
$results = process_delivery(array('hash' => $contact['xchan_hash']), get_item_elements($message), array(array('hash' => $importer['xchan_hash'])), false);
logger('onepoll: feed_update: process_delivery: ' . print_r($results, true));
$total++;
}
logger("onepoll: {$total} messages processed");
}
}
}
// fetch some items
// set last updated timestamp
if ($contact['xchan_connurl']) {
$r = q("SELECT xlink_id from xlink \n\t\t\twhere xlink_xchan = '%s' and xlink_updated > UTC_TIMESTAMP() - INTERVAL 1 DAY limit 1", intval($contact['xchan_hash']));
if (!$r) {
poco_load($contact['xchan_hash'], $contact['xchan_connurl']);
}
}
return;
}