function queue_run($argv, $argc) { cli_startup(); global $a; require_once 'include/items.php'; require_once 'include/bbcode.php'; if (argc() > 1) { $queue_id = argv(1); } else { $queue_id = 0; } $deadguys = array(); logger('queue: start'); $r = q("DELETE FROM outq WHERE outq_created < UTC_TIMESTAMP() - INTERVAL 3 DAY"); if ($queue_id) { $r = q("SELECT * FROM outq WHERE outq_hash = '%s' LIMIT 1", dbesc($queue_id)); } else { // For the first 12 hours we'll try to deliver every 15 minutes // After that, we'll only attempt delivery once per hour. // This currently only handles the default queue drivers ('zot' or '') which we will group by posturl // so that we don't start off a thousand deliveries for a couple of dead hubs. // The zot driver will deliver everything destined for a single hub once contact is made (*if* contact is made). // Other drivers will have to do something different here and may need their own query. $r = q("SELECT * FROM outq WHERE outq_delivered = 0 and (( outq_created > UTC_TIMESTAMP() - INTERVAL 12 HOUR and outq_updated < UTC_TIMESTAMP() - INTERVAL 15 MINUTE ) OR ( outq_updated < UTC_TIMESTAMP() - INTERVAL 1 HOUR )) group by outq_posturl"); } if (!$r) { return; } foreach ($r as $rr) { if (in_array($rr['outq_posturl'], $deadguys)) { continue; } if ($rr['outq_driver'] === 'post') { $result = z_post_url($rr['outq_posturl'], $rr['outq_msg']); if ($result['success'] && $result['return_code'] < 300) { logger('queue: queue post success to ' . $rr['outq_posturl'], LOGGER_DEBUG); $y = q("delete from outq where outq_hash = '%s' limit 1", dbesc($rr['ouq_hash'])); } else { logger('queue: queue post returned ' . $result['return_code'] . ' from ' . $rr['outq_posturl'], LOGGER_DEBUG); $y = q("update outq set outq_updated = '%s' where outq_hash = '%s' limit 1", dbesc(datetime_convert()), dbesc($rr['outq_hash'])); } continue; } $result = zot_zot($rr['outq_posturl'], $rr['outq_notify']); if ($result['success']) { zot_process_response($rr['outq_posturl'], $result, $rr); } else { $deadguys[] = $rr['outq_posturl']; $y = q("update outq set outq_updated = '%s' where outq_hash = '%s' limit 1", dbesc(datetime_convert()), dbesc($rr['outq_hash'])); } } }
function deliver_run($argv, $argc) { cli_startup(); $a = get_app(); if ($argc < 2) { return; } logger('deliver: invoked: ' . print_r($argv, true), LOGGER_DATA); for ($x = 1; $x < $argc; $x++) { $r = q("select * from outq where outq_hash = '%s' limit 1", dbesc($argv[$x])); if ($r) { if ($r[0]['outq_driver'] === 'post') { $result = z_post_url($r[0]['outq_posturl'], $r[0]['outq_msg']); if ($result['success'] && $result['return_code'] < 300) { logger('deliver: queue post success to ' . $r[0]['outq_posturl'], LOGGER_DEBUG); $y = q("delete from outq where outq_hash = '%s' limit 1", dbesc($argv[$x])); } else { logger('deliver: queue post returned ' . $result['return_code'] . ' from ' . $r[0]['outq_posturl'], LOGGER_DEBUG); $y = q("update outq set outq_updated = '%s' where outq_hash = '%s' limit 1", dbesc(datetime_convert()), dbesc($argv[$x])); } continue; } if ($r[0]['outq_posturl'] === z_root() . '/post') { logger('deliver: local delivery', LOGGER_DEBUG); // local delivery // we should probably batch these and save a few delivery processes // If there is no outq_msg, this is a refresh_all message which does not require local handling if ($r[0]['outq_msg']) { $msg = array('body' => json_encode(array('pickup' => array(array('notify' => json_decode($r[0]['outq_notify'], true), 'message' => json_decode($r[0]['outq_msg'], true)))))); zot_import($msg, z_root()); $r = q("delete from outq where outq_hash = '%s' limit 1", dbesc($argv[$x])); } } else { logger('deliver: dest: ' . $r[0]['outq_posturl'], LOGGER_DEBUG); $result = zot_zot($r[0]['outq_posturl'], $r[0]['outq_notify']); if ($result['success']) { zot_process_response($r[0]['outq_posturl'], $result, $r[0]); } else { $y = q("update outq set outq_updated = '%s' where outq_hash = '%s' limit 1", dbesc(datetime_convert()), dbesc($argv[$x])); } } } } }
function queue_deliver($outq, $immediate = false) { $base = null; $h = parse_url($outq['outq_posturl']); if ($h) { $base = $h['scheme'] . '://' . $h['host'] . ($h['port'] ? ':' . $h['port'] : ''); } if ($base && $base !== z_root() && $immediate) { $y = q("select site_update, site_dead from site where site_url = '%s' ", dbesc($base)); if ($y) { if (intval($y[0]['site_dead'])) { remove_queue_by_posturl($outq['outq_posturl']); logger('dead site ignored ' . $base); return; } if ($y[0]['site_update'] < datetime_convert('UTC', 'UTC', 'now - 1 month')) { update_queue_item($outq['outq_hash'], 10); logger('immediate delivery deferred for site ' . $base); return; } } else { // zot sites should all have a site record, unless they've been dead for as long as // your site has existed. Since we don't know for sure what these sites are, // call them unknown q("insert into site (site_url, site_update, site_dead, site_type) values ('%s','%s',0,%d) ", dbesc($base), dbesc(datetime_convert()), intval($outq['outq_driver'] === 'post' ? SITE_TYPE_NOTZOT : SITE_TYPE_UNKNOWN)); } } $arr = array('outq' => $outq, 'base' => $base, 'handled' => false, 'immediate' => $immediate); call_hooks('queue_deliver', $arr); if ($arr['handled']) { return; } // "post" queue driver - used for diaspora and friendica-over-diaspora communications. if ($outq['outq_driver'] === 'post') { $result = z_post_url($outq['outq_posturl'], $outq['outq_msg']); if ($result['success'] && $result['return_code'] < 300) { logger('deliver: queue post success to ' . $outq['outq_posturl'], LOGGER_DEBUG); if ($base) { q("update site set site_update = '%s', site_dead = 0 where site_url = '%s' ", dbesc(datetime_convert()), dbesc($base)); } q("update dreport set dreport_result = '%s', dreport_time = '%s' where dreport_queue = '%s' limit 1", dbesc('accepted for delivery'), dbesc(datetime_convert()), dbesc($outq['outq_hash'])); remove_queue_item($outq['outq_hash']); // server is responding - see if anything else is going to this destination and is piled up // and try to send some more. We're relying on the fact that delivery_loop() results in an // immediate delivery otherwise we could get into a queue loop. if (!$immediate) { $x = q("select outq_hash from outq where outq_posturl = '%s' and outq_delivered = 0", dbesc($outq['outq_posturl'])); $piled_up = array(); if ($x) { foreach ($x as $xx) { $piled_up[] = $xx['outq_hash']; } } if ($piled_up) { delivery_loop($piled_up); } } } else { logger('deliver: queue post returned ' . $result['return_code'] . ' from ' . $outq['outq_posturl'], LOGGER_DEBUG); update_queue_item($outq['outq_posturl']); } return; } // normal zot delivery logger('deliver: dest: ' . $outq['outq_posturl'], LOGGER_DEBUG); $result = zot_zot($outq['outq_posturl'], $outq['outq_notify']); if ($result['success']) { logger('deliver: remote zot delivery succeeded to ' . $outq['outq_posturl']); zot_process_response($outq['outq_posturl'], $result, $outq); } else { logger('deliver: remote zot delivery failed to ' . $outq['outq_posturl']); logger('deliver: remote zot delivery fail data: ' . print_r($result, true), LOGGER_DATA); update_queue_item($outq['outq_hash'], 10); } return; }
function queue_run($argv, $argc) { cli_startup(); global $a; require_once 'include/items.php'; require_once 'include/bbcode.php'; if (argc() > 1) { $queue_id = argv(1); } else { $queue_id = 0; } $deadguys = array(); logger('queue: start'); $r = q("select outq_posturl from outq where outq_created < %s - INTERVAL %s", db_utcnow(), db_quoteinterval('3 DAY')); if ($r) { foreach ($r as $rr) { $site_url = ''; $h = parse_url($rr['outq_posturl']); $desturl = $h['scheme'] . '://' . $h['host'] . ($h['port'] ? ':' . $h['port'] : ''); q("update site set site_dead = 1 where site_dead = 0 and site_url = '%s' and site_update < %s - INTERVAL %s", dbesc($desturl), db_utcnow(), db_quoteinterval('1 MONTH')); } } $r = q("DELETE FROM outq WHERE outq_created < %s - INTERVAL %s", db_utcnow(), db_quoteinterval('3 DAY')); if ($queue_id) { $r = q("SELECT * FROM outq WHERE outq_hash = '%s' LIMIT 1", dbesc($queue_id)); } else { // For the first 12 hours we'll try to deliver every 15 minutes // After that, we'll only attempt delivery once per hour. // This currently only handles the default queue drivers ('zot' or '') which we will group by posturl // so that we don't start off a thousand deliveries for a couple of dead hubs. // The zot driver will deliver everything destined for a single hub once contact is made (*if* contact is made). // Other drivers will have to do something different here and may need their own query. // Note: this requires some tweaking as new posts to long dead hubs once a day will keep them in the // "every 15 minutes" category. We probably need to prioritise them when inserted into the queue // or just prior to this query based on recent and long-term delivery history. If we have good reason to believe // the site is permanently down, there's no reason to attempt delivery at all, or at most not more than once // or twice a day. // FIXME: can we sort postgres on outq_priority and maintain the 'distinct' ? // The order by max(outq_priority) might be a dodgy query because of the group by. // The desired result is to return a sequence in the order most likely to be delivered in this run. // If a hub has already been sitting in the queue for a few days, they should be delivered last; // hence every failure should drop them further down the priority list. if (ACTIVE_DBTYPE == DBTYPE_POSTGRES) { $prefix = 'DISTINCT ON (outq_posturl)'; $suffix = 'ORDER BY outq_posturl'; } else { $prefix = ''; $suffix = 'GROUP BY outq_posturl ORDER BY max(outq_priority)'; } $r = q("SELECT {$prefix} * FROM outq WHERE outq_delivered = 0 and (( outq_created > %s - INTERVAL %s and outq_updated < %s - INTERVAL %s ) OR ( outq_updated < %s - INTERVAL %s )) {$suffix}", db_utcnow(), db_quoteinterval('12 HOUR'), db_utcnow(), db_quoteinterval('15 MINUTE'), db_utcnow(), db_quoteinterval('1 HOUR')); } if (!$r) { return; } foreach ($r as $rr) { if (in_array($rr['outq_posturl'], $deadguys)) { continue; } if ($rr['outq_driver'] === 'post') { $result = z_post_url($rr['outq_posturl'], $rr['outq_msg']); if ($result['success'] && $result['return_code'] < 300) { logger('queue: queue post success to ' . $rr['outq_posturl'], LOGGER_DEBUG); $y = q("delete from outq where outq_hash = '%s'", dbesc($rr['ouq_hash'])); } else { logger('queue: queue post returned ' . $result['return_code'] . ' from ' . $rr['outq_posturl'], LOGGER_DEBUG); $y = q("update outq set outq_updated = '%s', outq_priority = outq_priority + 10 where outq_hash = '%s'", dbesc(datetime_convert()), dbesc($rr['outq_hash'])); } continue; } $result = zot_zot($rr['outq_posturl'], $rr['outq_notify']); if ($result['success']) { logger('queue: deliver zot success to ' . $rr['outq_posturl'], LOGGER_DEBUG); zot_process_response($rr['outq_posturl'], $result, $rr); } else { $deadguys[] = $rr['outq_posturl']; logger('queue: deliver zot returned ' . $result['return_code'] . ' from ' . $rr['outq_posturl'], LOGGER_DEBUG); $y = q("update outq set outq_updated = '%s', outq_priority = outq_priority + 10 where outq_hash = '%s'", dbesc(datetime_convert()), dbesc($rr['outq_hash'])); } } }
function deliver_run($argv, $argc) { cli_startup(); $a = get_app(); if ($argc < 2) { return; } logger('deliver: invoked: ' . print_r($argv, true), LOGGER_DATA); for ($x = 1; $x < $argc; $x++) { $r = q("select * from outq where outq_hash = '%s' limit 1", dbesc($argv[$x])); if ($r) { /** * Check to see if we have any recent communications with this hub (in the last month). * If not, reduce the outq_priority. */ $h = parse_url($r[0]['outq_posturl']); if ($h) { $base = $h['scheme'] . '://' . $h['host'] . ($h['port'] ? ':' . $h['port'] : ''); if ($base !== z_root()) { $y = q("select site_update, site_dead from site where site_url = '%s' ", dbesc($base)); if ($y) { if (intval($y[0]['site_dead'])) { q("delete from outq where outq_posturl = '%s'", dbesc($r[0]['outq_posturl'])); logger('dead site ignored ' . $base); continue; } if ($y[0]['site_update'] < datetime_convert('UTC', 'UTC', 'now - 1 month')) { q("update outq set outq_priority = %d where outq_hash = '%s'", intval($r[0]['outq_priority'] + 10), dbesc($r[0]['outq_hash'])); logger('immediate delivery deferred for site ' . $base); continue; } } } } // "post" queue driver - used for diaspora and friendica-over-diaspora communications. if ($r[0]['outq_driver'] === 'post') { $result = z_post_url($r[0]['outq_posturl'], $r[0]['outq_msg']); if ($result['success'] && $result['return_code'] < 300) { logger('deliver: queue post success to ' . $r[0]['outq_posturl'], LOGGER_DEBUG); $y = q("delete from outq where outq_hash = '%s'", dbesc($argv[$x])); } else { logger('deliver: queue post returned ' . $result['return_code'] . ' from ' . $r[0]['outq_posturl'], LOGGER_DEBUG); $y = q("update outq set outq_updated = '%s' where outq_hash = '%s'", dbesc(datetime_convert()), dbesc($argv[$x])); } continue; } $notify = json_decode($r[0]['outq_notify'], true); // Check if this is a conversation request packet. It won't have outq_msg // but will be an encrypted packet - so will need to be handed off to // web delivery rather than processed inline. $sendtoweb = false; if (array_key_exists('iv', $notify) && !$r[0]['outq_msg']) { $sendtoweb = true; } if ($r[0]['outq_posturl'] === z_root() . '/post' && !$sendtoweb) { logger('deliver: local delivery', LOGGER_DEBUG); // local delivery // we should probably batch these and save a few delivery processes if ($r[0]['outq_msg']) { $m = json_decode($r[0]['outq_msg'], true); if (array_key_exists('message_list', $m)) { foreach ($m['message_list'] as $mm) { $msg = array('body' => json_encode(array('success' => true, 'pickup' => array(array('notify' => $notify, 'message' => $mm))))); zot_import($msg, z_root()); } } else { $msg = array('body' => json_encode(array('success' => true, 'pickup' => array(array('notify' => $notify, 'message' => $m))))); zot_import($msg, z_root()); } $r = q("delete from outq where outq_hash = '%s'", dbesc($argv[$x])); } } else { logger('deliver: dest: ' . $r[0]['outq_posturl'], LOGGER_DEBUG); $result = zot_zot($r[0]['outq_posturl'], $r[0]['outq_notify']); if ($result['success']) { logger('deliver: remote zot delivery succeeded to ' . $r[0]['outq_posturl']); zot_process_response($r[0]['outq_posturl'], $result, $r[0]); } else { logger('deliver: remote zot delivery failed to ' . $r[0]['outq_posturl']); $y = q("update outq set outq_updated = '%s' where outq_hash = '%s'", dbesc(datetime_convert()), dbesc($argv[$x])); } } } } }
function deliver_run($argv, $argc) { cli_startup(); $a = get_app(); if ($argc < 2) { return; } logger('deliver: invoked: ' . print_r($argv, true), LOGGER_DATA); for ($x = 1; $x < $argc; $x++) { $dresult = null; $r = q("select * from outq where outq_hash = '%s' limit 1", dbesc($argv[$x])); if ($r) { /** * Check to see if we have any recent communications with this hub (in the last month). * If not, reduce the outq_priority. */ $h = parse_url($r[0]['outq_posturl']); if ($h) { $base = $h['scheme'] . '://' . $h['host'] . ($h['port'] ? ':' . $h['port'] : ''); if ($base !== z_root()) { $y = q("select site_update, site_dead from site where site_url = '%s' ", dbesc($base)); if ($y) { if (intval($y[0]['site_dead'])) { q("delete from outq where outq_posturl = '%s'", dbesc($r[0]['outq_posturl'])); logger('dead site ignored ' . $base); continue; } if ($y[0]['site_update'] < datetime_convert('UTC', 'UTC', 'now - 1 month')) { q("update outq set outq_priority = %d where outq_hash = '%s'", intval($r[0]['outq_priority'] + 10), dbesc($r[0]['outq_hash'])); logger('immediate delivery deferred for site ' . $base); continue; } } else { // zot sites should all have a site record, unless they've been dead for as long as // your site has existed. Since we don't know for sure what these sites are, // call them unknown q("insert into site (site_url, site_update, site_dead, site_type) values ('%s','%s',0,%d) ", dbesc($base), dbesc(datetime_convert()), intval($r[0]['outq_driver'] === 'post' ? SITE_TYPE_NOTZOT : SITE_TYPE_UNKNOWN)); } } } // "post" queue driver - used for diaspora and friendica-over-diaspora communications. if ($r[0]['outq_driver'] === 'post') { $result = z_post_url($r[0]['outq_posturl'], $r[0]['outq_msg']); if ($result['success'] && $result['return_code'] < 300) { logger('deliver: queue post success to ' . $r[0]['outq_posturl'], LOGGER_DEBUG); q("update site set site_update = '%s', site_dead = 0 where site_url = '%s' ", dbesc(datetime_convert()), dbesc($site_url)); q("update dreport set status = '%s', dreport_time = '%s' where dreport_queue = '%s' limit 1", dbesc('accepted for delivery'), dbesc(datetime_convert()), dbesc($argv[$x])); $y = q("delete from outq where outq_hash = '%s'", dbesc($argv[$x])); } else { logger('deliver: queue post returned ' . $result['return_code'] . ' from ' . $r[0]['outq_posturl'], LOGGER_DEBUG); $y = q("update outq set outq_updated = '%s' where outq_hash = '%s'", dbesc(datetime_convert()), dbesc($argv[$x])); } continue; } $notify = json_decode($r[0]['outq_notify'], true); // Check if this is a conversation request packet. It won't have outq_msg // but will be an encrypted packet - so will need to be handed off to // web delivery rather than processed inline. $sendtoweb = false; if (array_key_exists('iv', $notify) && !$r[0]['outq_msg']) { $sendtoweb = true; } if ($r[0]['outq_posturl'] === z_root() . '/post' && !$sendtoweb) { logger('deliver: local delivery', LOGGER_DEBUG); // local delivery // we should probably batch these and save a few delivery processes if ($r[0]['outq_msg']) { $m = json_decode($r[0]['outq_msg'], true); if (array_key_exists('message_list', $m)) { foreach ($m['message_list'] as $mm) { $msg = array('body' => json_encode(array('success' => true, 'pickup' => array(array('notify' => $notify, 'message' => $mm))))); zot_import($msg, z_root()); } } else { $msg = array('body' => json_encode(array('success' => true, 'pickup' => array(array('notify' => $notify, 'message' => $m))))); $dresult = zot_import($msg, z_root()); } $r = q("delete from outq where outq_hash = '%s'", dbesc($argv[$x])); if ($dresult && is_array($dresult)) { foreach ($dresult as $xx) { if (is_array($xx) && array_key_exists('message_id', $xx)) { q("insert into dreport ( dreport_mid, dreport_site, dreport_recip, dreport_result, dreport_time, dreport_xchan ) values ( '%s', '%s','%s','%s','%s','%s' ) ", dbesc($xx['message_id']), dbesc($xx['location']), dbesc($xx['recipient']), dbesc($xx['status']), dbesc(datetime_convert($xx['date'])), dbesc($xx['sender'])); } } } q("delete from dreport where dreport_queue = '%s' limit 1", dbesc($argv[$x])); } } else { logger('deliver: dest: ' . $r[0]['outq_posturl'], LOGGER_DEBUG); $result = zot_zot($r[0]['outq_posturl'], $r[0]['outq_notify']); if ($result['success']) { logger('deliver: remote zot delivery succeeded to ' . $r[0]['outq_posturl']); zot_process_response($r[0]['outq_posturl'], $result, $r[0]); } else { logger('deliver: remote zot delivery failed to ' . $r[0]['outq_posturl']); $y = q("update outq set outq_updated = '%s' where outq_hash = '%s'", dbesc(datetime_convert()), dbesc($argv[$x])); } } } } }