public function processEach($proc_f)
{
$row_i = 0;
$this->db->begin();
while (($row = $this->getRow($row_i)) !== false) {
/* it's important that we iterate the mapping to preserve key order */
$mapping = $this->mapping;
$headers = $this->headers;
$params = new DTParams(array_reduce(array_keys($mapping), function ($out, $k) use($mapping, $headers, $row) {
$i = $headers[$k];
$out[$mapping[$k]] = isset($row[$i]) ? $row[$i] : null;
return $out;
}, array()));
$mapped = $params->allParams();
try {
$proc_f($mapped, $params);
} catch (Exception $e) {
DTLog::debug("Failed to ingest (%s):\n%s", $e->getMessage(), $mapped);
}
$row_i++;
if ($row_i % $this->transaction_size == 0) {
if ($this->dry_run) {
$this->db->rollback();
} else {
$this->db->commit();
}
$this->db->begin();
DTLog::debug("* ({$row_i})");
}
}
if ($this->dry_run) {
$this->db->rollback();
} else {
$this->db->commit();
}
}
public function synchronizeItems(&$version, $prefix = null)
{
$count = 0;
$prefix = empty($prefix) ? "users/{$this->user}" : $prefix;
$items = $this->itemsList($version, $prefix);
DTLog::info("synchronizing %s items...", count($items));
$highest_version = 0;
foreach ($items as $itemKey => $item_version) {
//step through the modified items
$count++;
if ($count % 20 == 0) {
DTLog::info("> %s%%", round($count / count($items) * 100));
}
$highest_version = max($highest_version, $item_version);
$item = $this->item($itemKey, $prefix);
if ($item == null || $item["itemType"] == "note" || $item["itemType"] == "attachment") {
//well, we didn't get it, what can we do?
continue;
}
$item["itemKey"] = $itemKey;
//we keep this for the delete logic (see +synchronize()+)
//DTLog::debug($item);
$params = new DTParams($item);
//give us the chance to clean the params (only once!)
$title = strtolower($params->stringParam("title"));
$itemKey = $params->stringParam("itemKey");
//$qb = $this->db->where("item_key='{$itemKey}' OR LEVENSHTEIN(LOWER(substring(title from 1 for 250)),substring('{$title}' from 1 for 250))<10");
$qb = $this->db->where("item_key='{$itemKey}' OR LEVENSHTEIN(LOWER(title),'{$title}')<6");
$this->mergePublication($qb, $params->allParams());
}
return $highest_version;
}
public function testJournalArticle()
{
$params = new DTParams($this->samples["journalArticle"]);
$clean = $params->allParams();
$pub = ZoteroPublication::upsert($this->db->qb()->fail(), $clean);
$item = new ZoteroPublication($this->db->filter(array("type_id" => 4)));
$this->assertEquals(15, $item["volume"]);
$this->assertEquals(3, $item["issue"]);
$this->assertEquals("54-61", $item["pages"]);
}
