<?php /* * Copyright (c) 2015, xrstf | MIT licensed */ require __DIR__ . '/boot.php'; $app = new xrstf\oeisApi\Application(); $client = new xrstf\oeisApi\OEISClient(); $db = $app['database']; $start = $db->fetch('SELECT MAX(id) FROM sequence') + 1; logLine('Scouting starts at ' . $start . '...', false); $a = microtime(true); $html = $client->fetch(range($start, $start + 9)); $time = microtime(true) - $a; if (!$html) { $error = error_get_last(); logLine(' scout died: ' . $error['message'], true, false); exit(1); } logLine(sprintf(' scout returned with %.2F KiB markup after %.2F s.', strlen($html) / 1024.0, $time), true, false); logLine('Parsing loot and searching sequences...', false); $parser = new xrstf\oeisApi\Sequence\Parser(); $importer = new xrstf\oeisApi\Sequence\Importer($db); $sequences = $parser->parseDocument($html); if ($sequences === false) { logLine(' error: parsing failed!', true, false); exit(1); } if (count($sequences) === 0) { logLine(' no new sequences found in this chunk :(', true, false); exit(1);
$workerID = 1; while (true) { $db->query('UPDATE sequence SET worker = %i WHERE worker IS NULL ORDER BY updated_at ASC, id ASC LIMIT 10', [$workerID]); $sequences = $db->fetchMap('SELECT id, imported_at FROM sequence WHERE worker = %i', [$workerID]); $sequenceIDs = array_keys($sequences); if (count($sequenceIDs) === 0) { logLine('Could not acquire any sequences to update. Waiting for another turn.'); sleep(3); continue; } $ids = array_map(function ($sequenceID) { return sprintf('A%06d', $sequenceID); }, $sequenceIDs); logLine('Fetching sequences ' . str_replace('id:', '', implode(', ', $ids)) . '...', false); $a = microtime(true); $html = $client->fetch($sequenceIDs); $time = microtime(true) - $a; if (!$html) { $error = error_get_last(); logLine(' error: HTTP request failed: ' . $error['message'], true, false); bail($db); sleep(3); continue; } logLine(sprintf(' fetched %.2F KiB markup in %.2F s.', strlen($html) / 1024.0, $time), true, false); logLine('Parsing document and searching sequences...', false); $parser = new xrstf\oeisApi\Sequence\Parser(); $importer = new xrstf\oeisApi\Sequence\Importer($db); $sequences = $parser->parseDocument($html); if ($sequences === false) { logLine(' error: parsing failed!', true, false);