public function processEach($proc_f) { $row_i = 0; $this->db->begin(); while (($row = $this->getRow($row_i)) !== false) { /* it's important that we iterate the mapping to preserve key order */ $mapping = $this->mapping; $headers = $this->headers; $params = new DTParams(array_reduce(array_keys($mapping), function ($out, $k) use($mapping, $headers, $row) { $i = $headers[$k]; $out[$mapping[$k]] = isset($row[$i]) ? $row[$i] : null; return $out; }, array())); $mapped = $params->allParams(); try { $proc_f($mapped, $params); } catch (Exception $e) { DTLog::debug("Failed to ingest (%s):\n%s", $e->getMessage(), $mapped); } $row_i++; if ($row_i % $this->transaction_size == 0) { if ($this->dry_run) { $this->db->rollback(); } else { $this->db->commit(); } $this->db->begin(); DTLog::debug("* ({$row_i})"); } } if ($this->dry_run) { $this->db->rollback(); } else { $this->db->commit(); } }
public function synchronizeItems(&$version, $prefix = null) { $count = 0; $prefix = empty($prefix) ? "users/{$this->user}" : $prefix; $items = $this->itemsList($version, $prefix); DTLog::info("synchronizing %s items...", count($items)); $highest_version = 0; foreach ($items as $itemKey => $item_version) { //step through the modified items $count++; if ($count % 20 == 0) { DTLog::info("> %s%%", round($count / count($items) * 100)); } $highest_version = max($highest_version, $item_version); $item = $this->item($itemKey, $prefix); if ($item == null || $item["itemType"] == "note" || $item["itemType"] == "attachment") { //well, we didn't get it, what can we do? continue; } $item["itemKey"] = $itemKey; //we keep this for the delete logic (see +synchronize()+) //DTLog::debug($item); $params = new DTParams($item); //give us the chance to clean the params (only once!) $title = strtolower($params->stringParam("title")); $itemKey = $params->stringParam("itemKey"); //$qb = $this->db->where("item_key='{$itemKey}' OR LEVENSHTEIN(LOWER(substring(title from 1 for 250)),substring('{$title}' from 1 for 250))<10"); $qb = $this->db->where("item_key='{$itemKey}' OR LEVENSHTEIN(LOWER(title),'{$title}')<6"); $this->mergePublication($qb, $params->allParams()); } return $highest_version; }
public function testJournalArticle() { $params = new DTParams($this->samples["journalArticle"]); $clean = $params->allParams(); $pub = ZoteroPublication::upsert($this->db->qb()->fail(), $clean); $item = new ZoteroPublication($this->db->filter(array("type_id" => 4))); $this->assertEquals(15, $item["volume"]); $this->assertEquals(3, $item["issue"]); $this->assertEquals("54-61", $item["pages"]); }