public function insertDaysData($datestamp = null)
{
$articles = array();
$this->batchArray = array();
$this->batchCount = 0;
echo "Starting " . __METHOD__ . " at " . wfTimestamp(TS_MW) . "\n";
$beginTime = microtime(true);
$titus = new TitusDB();
if ($datestamp == null) {
$rows = $titus->getRecords();
} else {
$rows = $titus->getOldRecords($datestamp);
}
foreach ($rows as $row) {
$newRow = (array) $row;
$newRow['page_id_language'] = $newRow['ti_language_code'] . $newRow['ti_page_id'];
//need this to actually be an integer for DynamoDB
$articles[] = $newRow;
}
echo "Putting " . count($articles) . " rows into dynamodb\n";
$this->batchArray[] = array();
$this->batchCount++;
foreach ($articles as $articleData) {
$this->addDataToBatch($articleData, "PutRequest");
}
echo "Getting ready to process " . $this->batchCount . " batches\n\n";
$errorCount = 0;
for ($i = 0; $i < $this->batchCount; $i++) {
$startTime = microtime(true);
$this->processBatch($this->batchArray[$i], "PutRequest", $errorCount);
$finishTime = microtime(true);
echo "batch #{$i} of {$this->batchCount} " . ($finishTime - $startTime) . "s, ";
if ($errorCount >= $this->maxErrors) {
echo "Too many errors found. Stopping this run.\n\n";
mail('*****@*****.**', 'Dynamo Quitting', 'Quitting Dynamo run...too many errors.');
break;
}
while ($finishTime - $startTime < 1) {
//need to wait till a second is up.
$finishTime = microtime(true);
}
$currentTime = microtime(true);
if ($currentTime - $beginTime > $this->maxTime) {
echo "Taking too long. Stopping this run.\n\n";
mail('*****@*****.**', 'Dynamo Quitting', 'Quitting Dynamo run...taking too long.');
break;
}
}
echo "\nFinished " . __METHOD__ . " at " . wfTimestamp(TS_MW) . " for a total time of " . ($currentTime - $beginTime) . " seconds.\n";
}
