if (preg_match("/^insert/i", $x)) {
print mysql_affected_rows() . " INSERTED\n \n";
}
}
}
}
return $res;
}
require_once dirname(__FILE__) . "/../../common/connect/applyCredentials.php";
require_once dirname(__FILE__) . "/../../common/php/dbMySqlWrappers.php";
require_once dirname(__FILE__) . "/../../common/php/saveRecord.php";
require_once dirname(__FILE__) . "/../../common/php/utilsTitleMask.php";
// 26/3/14 Functions to index record being saved using Elastic Search (Lucene)
require_once dirname(__FILE__) . "/../../records/index/elasticSearchFunctions.php";
if (!is_logged_in()) {
jsonError("no logged-in user");
}
$_REQUEST = json_decode(@$_POST["data"] ? $_POST["data"] : base64_decode(@$_GET["data"]), true);
mysql_connection_overwrite(DATABASE);
mysql_query("start transaction");
$out = saveRecord(@$_REQUEST["id"], @$_REQUEST["type"], @$_REQUEST["url"], @$_REQUEST["notes"], @$_REQUEST["group"], @$_REQUEST["vis"], @$_REQUEST["bookmark"], @$_REQUEST["pnotes"], @$_REQUEST["rating"], @$_REQUEST["tags"], @$_REQUEST["wgTags"], @$_REQUEST["detail"], @$_REQUEST["-notify"], @$_REQUEST["+notify"], @$_REQUEST["-comment"], @$_REQUEST["comment"], @$_REQUEST["+comment"]);
mysql_query("commit");
// 26/3/14 Add record to index in Elastic Search (Lucene)
updateRecordIndexEntry(HEURIST_DBNAME, @$_REQUEST["type"], @$_REQUEST["id"]);
print json_format($out);
function jsonError($message)
{
mysql_query("rollback");
print "{\"error\":\"" . addslashes($message) . "\"}";
exit(0);
}
/**
*
* @global type $rep_counter
* @global string $rep_issues
* @global array $fieldhelper_to_heurist_map
* @param type $dir
*/
function doHarvestInDir($dir)
{
global $rep_issues, $fieldhelper_to_heurist_map, $mediaExts, $progress_divid, $geoDT, $fileDT, $titleDT, $startdateDT, $enddateDT, $descriptionDT;
$rep_processed = 0;
$rep_processed_dir = 0;
$rep_ignored = 0;
$f_items = null;
//reference to items element
$progress_divid++;
print "<div><b>{$dir}</b><span id='progress{$progress_divid}'></span></div>";
ob_flush();
flush();
if (!is_writable($dir)) {
//$rep_issues = "Folder ".$dir." is not writable. Check permissions";
print "<div style=\"color:red\">Folder is not writeable. Check permissions</div>";
return 0;
}
$manifest_file = $dir . "fieldhelper.xml";
$all_files = scandir($dir);
/*****DEBUG****/
//
if (file_exists($manifest_file)) {
//read fieldhelpe.xml
if (is_readable($manifest_file)) {
//check write permission
if (!is_writable($manifest_file)) {
print "<div style=\"color:red\">Manifest is not writable. Check permissions</div>";
//$rep_issues = $rep_issues."<br/> Manifest is not writable in ".$dir;
return 0;
}
} else {
print "<div style=\"color:red\">Manifest is not readable. Check permissions</div>";
//$rep_issues = $rep_issues."<br> manifest is not readable in ".$dir;
return 0;
}
$fh_data = simplexml_load_file($manifest_file);
if ($fh_data == null || is_string($fh_data)) {
print "<div style=\"color:red\">Manifest is corrupted</div>";
//$rep_issues = "Manifest file is corrupted";
return 0;
}
//MAIN LOOP in manifest
$not_found = true;
foreach ($fh_data->children() as $f_gen) {
if ($f_gen->getName() == "items") {
$f_items = $f_gen;
$not_found = false;
$tot_files = count($f_gen->children());
$cnt_files = 0;
foreach ($f_gen->children() as $f_item) {
$recordId = null;
$recordType = RT_MEDIA_RECORD;
//media by default
$recordURL = null;
$recordNotes = null;
$el_heuristid = null;
$lat = null;
$lon = null;
$filename = null;
$filename_base = null;
$details = array();
$file_id = null;
$old_md5 = null;
foreach ($f_item->children() as $el) {
//$key=>$value
$content = strval($el);
// (string)$el;
$key = $el->getName();
$value = $content;
/*foreach ($el as $key=>$value2){
$value = $value2;
break;
}*/
if ($key == "md5") {
$old_md5 = $value;
} else {
if (array_key_exists($key, $fieldhelper_to_heurist_map) && $fieldhelper_to_heurist_map[$key]) {
$key2 = $fieldhelper_to_heurist_map[$key];
if ($key2 == "file") {
$filename = $dir . $value;
$filename_base = $value;
$key3 = $fieldhelper_to_heurist_map['file_name'];
if ($key3 > 0) {
$details["t:" . $key3] = array("1" => $value);
}
$key3 = $fieldhelper_to_heurist_map['file_path'];
if ($key3 > 0) {
$details["t:" . $key3] = array("1" => $dir);
}
} else {
if ($key2 == "lat") {
$lat = floatval($value);
} else {
if ($key2 == "lon") {
$lon = floatval($value);
} else {
if ($key2 == "recordId") {
$recordId = $value;
$el_heuristid = $el;
} else {
if (intval($key2) > 0) {
//add to details
$details["t:" . $key2] = array("1" => $value);
}
}
}
}
}
// else field type not defined in this instance
}
}
}
//for item keys
if ($filename) {
//exclude from the list of all files in this folder
if (in_array($filename_base, $all_files)) {
$ind = array_search($filename_base, $all_files, true);
unset($all_files[$ind]);
}
}
if ($recordId == null) {
//import only new
if ($filename) {
if (file_exists($filename)) {
$currfile = $filename;
//assign to global
//add-update the uploaded file
$file_id = register_file($filename, null, false);
if (is_numeric($file_id)) {
$details["t:" . $fileDT] = array("1" => $file_id);
} else {
print "<div style=\"color:#ff8844\">warning {$filename_base} failed to register, no record created</div>";
//$rep_issues = $rep_issues."<br/>Can't register file:".$filename.". ".$file_id;
$file_id = null;
}
} else {
print "<div style=\"color:#ff8844\">warning {$filename_base} file not found, no record created</div>";
}
}
if (!$file_id) {
continue;
//add with valid file only
}
if (is_numeric($lat) && is_numeric($lon) && ($lat != 0 || $lon != 0)) {
$details["t:" . $geoDT] = array("1" => "p POINT({$lon} {$lat})");
}
//set title by default
if (!array_key_exists("t:" . $titleDT, $details)) {
$details["t:" . $titleDT] = array("1" => $filename);
print "<div style=\"color:#ff8844\">warning {$filename_base} no title</div>";
}
$new_md5 = null;
$key = $fieldhelper_to_heurist_map['md5'];
if ($key > 0) {
$new_md5 = md5_file($filename);
$details["t:" . $key] = array("1" => $new_md5);
}
//add-update Heurist record
$out = saveRecord($recordId, $recordType, $recordURL, $recordNotes, null, null, null, null, null, null, null, $details, null, null, null, null, null);
if (@$out['error']) {
print "<div style='color:red'>{$filename_base} Error: " . implode("; ", $out["error"]) . "</div>";
} else {
if ($new_md5 == null) {
$new_md5 = md5_file($filename);
}
//update xml
if ($recordId == null) {
if ($old_md5 != $new_md5) {
print "<div style=\"color:#ff8844\">warning {$filename_base} checksum differs from value in manifest</div>";
}
$f_item->addChild("heurist_id", $out["bibID"]);
$f_item->addChild("md5", $new_md5);
$f_item->addChild("filesize", filesize($filename));
} else {
$el_heuristid["heurist_id"] = $out["bibID"];
}
if (@$out['warning']) {
print "<div style=\"color:#ff8844\">{$filename_base} Warning: " . implode("; ", $out["warning"]) . "</div>";
}
$rep_processed++;
}
} else {
$rep_ignored++;
}
$cnt_files++;
if ($cnt_files % 5 == 0) {
print '<script type="text/javascript">update_counts(' . $progress_divid . ',' . $cnt_files . ',' . $rep_processed . ',' . $tot_files . ')</script>' . "\n";
ob_flush();
flush();
}
}
//for items
}
//if has items
}
//for all children in manifest
if ($not_found) {
print "<div style=\"color:red\">Manifest is either corrupted or empty</div>";
//$rep_issues=$rep_issues."<br>folder $dir cotains corrupted or empty manifest file";
} else {
if ($rep_processed > 0) {
print "<div>{$rep_processed} records created</div>";
}
if ($rep_ignored > 0) {
print "<div>{$rep_ignored} files already indexed</div>";
//$rep_issues=$rep_issues."<br> $rep_ignored entries in manifest are ignored for ".$dir;
}
}
} else {
//create empty manifest XML - TODO!!!!
$s_manifest = <<<XML
<?xml version="1.0" encoding="UTF-8"?>
<fieldhelper version="1">
<info>
<AppName>Heurist</AppName>
<AppVersion>v 3.0.0 2012-01-01</AppVersion>
<AppCopyright>© ArtEresearch, University of Sydney</AppCopyright>
<date></date>
</info>
<formatOutput>1</formatOutput></fieldhelper>
XML;
$fh_data = simplexml_load_string($s_manifest);
}
// add new empty items element
if ($f_items == null) {
$f_items = $fh_data->addChild("items");
}
$tot_files = count($all_files);
$cnt_files = 0;
$cnt_added = 0;
//for files in folder that are not specified in the directory
foreach ($all_files as $filename) {
if (!($filename == "." || $filename == ".." || is_dir($dir . $filename) || $filename == "fieldhelper.xml")) {
/*****DEBUG****/
//error_log("2>>>>".is_dir($dir.$filename)." ".$filename);
$filename_base = $filename;
$filename = $dir . $filename;
$currfile = $filename;
$flleinfo = pathinfo($filename);
//checks for allowed extensions
if (in_array(strtolower($flleinfo['extension']), $mediaExts)) {
$details = array();
$file_id = register_file($filename, null, false);
if (is_numeric($file_id)) {
$details["t:" . $fileDT] = array("1" => $file_id);
} else {
print "<div style=\"color:#ff8844\">warning {$filename_base} failed to register, no record created: .{$file_id}</div>";
//$rep_issues = $rep_issues."<br/>Can't register file:".$filename.". ".$file_id;
$file_id = null;
continue;
}
$details["t:" . $titleDT] = array("1" => $flleinfo['basename']);
/* TODO - extract these data from exif
$details["t:".$descriptionDT] = array("1"=>$file_id);
$details["t:".$startdateDT] = array("1"=>$file_id);
$details["t:".$enddateDT] = array("1"=>$file_id);
$details["t:".$geoDT] = array("1"=>$file_id);
*/
$new_md5 = md5_file($filename);
$key = $fieldhelper_to_heurist_map['md5'];
if ($key > 0) {
$details["t:" . $key] = array("1" => $new_md5);
}
$key = $fieldhelper_to_heurist_map['file_name'];
if ($key > 0) {
$details["t:" . $key] = array("1" => $flleinfo['basename']);
}
$key = $fieldhelper_to_heurist_map['file_path'];
if ($key > 0) {
$details["t:" . $key] = array("1" => $flleinfo['dirname']);
}
$key = $fieldhelper_to_heurist_map['extension'];
if ($key > 0) {
$details["t:" . $key] = array("1" => $flleinfo['extension']);
}
$key = $fieldhelper_to_heurist_map['filesize'];
if ($key > 0) {
$details["t:" . $key] = array("1" => filesize($filename));
}
//add-update Heurist record
$out = saveRecord(null, RT_MEDIA_RECORD, null, null, null, null, null, null, null, null, null, $details, null, null, null, null, null);
/*****DEBUG****/
//error_log(">>>>>".filemtime($filename)." ".date("Y/m/d H:i:s.", filemtime($filename)));
$f_item = $f_items->addChild("item");
$f_item->addChild("filename", $flleinfo['basename']);
$f_item->addChild("nativePath", $filename);
$f_item->addChild("folder", $flleinfo['dirname']);
$f_item->addChild("extension", $flleinfo['extension']);
//$f_item->addChild("DateTime", );
//$f_item->addChild("DateTimeOriginal", );
$f_item->addChild("filedate", date("Y/m/d H:i:s.", filemtime($filename)));
$f_item->addChild("typeContent", "image");
$f_item->addChild("device", "image");
$f_item->addChild("duration", "2000");
$f_item->addChild("original_metadata", "chk");
$f_item->addChild("Name0", $flleinfo['basename']);
$f_item->addChild("heurist_id", $out["bibID"]);
$f_item->addChild("md5", $new_md5);
$f_item->addChild("filesize", filesize($filename));
$rep_processed_dir++;
$cnt_added++;
}
//check ext
$cnt_files++;
if ($cnt_files % 5 == 0) {
print '<script type="text/javascript">update_counts(' . $progress_divid . ',' . $cnt_files . ',' . $cnt_added . ',' . $tot_files . ')</script>' . "\n";
ob_flush();
flush();
}
}
}
//for files in folder that are not specified in the directory
if ($rep_processed_dir > 0) {
print "<div style=\"color:green\">{$rep_processed_dir} records created (new entries added to manifest)</div>";
}
print '<script type="text/javascript">update_counts(' . $progress_divid . ',' . $cnt_files . ',' . $cnt_added . ',0)</script>' . "\n";
ob_flush();
flush();
if ($rep_processed + $rep_processed_dir > 0) {
//save modified xml (with updated heurist_id tags
$fh_data->formatOutput = true;
$fh_data->saveXML($manifest_file);
}
return $rep_processed + $rep_processed_dir;
}
function doInsertUpdateRecord($recordId, $params, $details, $id_field, $mode_output)
{
global $mysqli, $imp_session, $rep_processed, $rep_added, $rep_updated, $rep_skipped, $rep_permission, $wg_id, $rec_visibility;
$import_table = $imp_session['import_table'];
$recordType = @$params['sa_rectype'];
//$id_field = @$params['recid_field']; //record ID field is always defined explicitly
//check permission beforehand
if ($recordId > 0) {
$res = checkPermission($recordId, $wg_id);
if ($res !== true) {
// error message: $res;
$rep_permission++;
return null;
}
}
//add-update Heurist record
$out = saveRecord($recordId, $recordType, @$details["recordURL"], @$details["recordNotes"], $wg_id, $rec_visibility, null, null, null, null, null, $details, null, null, null, null, null);
//error_log('to Add '.print_r($details,true));
//error_log('ADD REC '.print_r($out,true));
if (@$out['error']) {
//special formatting
foreach ($out["error"] as $idx => $value) {
$value = str_replace(". You may need to make fields optional. Missing data", "", $value);
$k = strpos($value, "Missing data for Required field(s) in");
if ($k !== false) {
$value = "<span style='color:red'>" . substr($value, 0, $k + 37) . "</span>" . substr($value, $k + 37);
} else {
$value = "<span style='color:red'>" . $value . "</span>";
}
$out["error"][$idx] = $value;
}
if ($mode_output != 'array') {
foreach ($details['imp_id'] as $imp_id) {
print "<div><span style='color:red'>Line: " . $imp_id . ".</span> " . implode("; ", $out["error"]);
$res = get_import_value($imp_id, $import_table);
if (is_array($res)) {
$s = htmlspecialchars(implode(", ", $res));
print "<div style='padding-left:40px'>" . $s . "AAAA</div>";
}
print "</div>";
}
}
$rep_skipped++;
$out["bibID"] = null;
} else {
if ($recordId != $out["bibID"]) {
//}==null){
if ($id_field && ($recordId == null || $recordId > 0)) {
$updquery = "UPDATE " . $import_table . " SET " . $id_field . "=" . $out["bibID"] . " WHERE imp_id in (" . implode(",", $details['imp_id']) . ")";
if (!$mysqli->query($updquery) && $mode_output != 'array') {
print "<div style='color:red'>Cannot update import table (set record id " . $out["bibID"] . ") for lines:" . implode(",", $details['imp_id']) . "</div>";
}
}
$rep_added++;
} else {
$rep_updated++;
}
if (@$out['warning'] && $mode_output != 'array') {
print "<div style=\"color:#ff8844\">Warning: " . implode("; ", $out["warning"]) . "</div>";
}
}
return @$out["bibID"];
}
$_REQUEST["records"][$nonce]["id"] = -1;
} else {
$id = mysql_insert_id();
$_REQUEST["records"][$nonce]["id"] = $id;
}
} else {
array_push($out["record"], array("error" => " creating temporary record nonce = {$nonce} no rectype given", "record" => $record, "nonce" => $nonce));
$_REQUEST["records"][$nonce]["id"] = -1;
}
}
$nonces[$nonce] = $_REQUEST["records"][$nonce]["id"];
}
foreach ($_REQUEST["records"] as $nonce => $record) {
// FIXME? should we perhaps index these by the nonce
if ($nonces[$nonce] != -1) {
$savedRecord = saveRecord(@$record["id"], @$record["type"], @$record["url"], @$record["notes"], @$record["group"], @$record["vis"], @$record["bookmark"], @$record["pnotes"], @$record["rating"], @$record["tags"], @$record["wgTags"], @$record["detail"], @$record["-notify"], @$record["+notify"], @$record["-comment"], @$record["comment"], @$record["+comment"], $nonces, $retitleRecs);
if (@$savedRecord['error']) {
//there was an error so give more context info back
$savedRecord['record'] = $record;
$savedRecord['nonce'] = $nonce;
}
array_push($out["record"], $savedRecord);
}
}
/*****DEBUG****/
//error_log("updating titlemasks ");
if (count($retitleRecs) > 0) {
foreach ($retitleRecs as $id) {
// calculate title, do an update
$query = "select rty_TitleMask, rty_ID from defRecTypes left join Records on rty_ID=rec_RecTypeID where rec_ID = {$id}";
$res = mysql_query($query);
function insert_update_Record($recID, $rectype, $details, $faims_id)
{
global $cntUpdated, $cntInsterted, $mysqli;
if ($recID > 0) {
//delete existing details
$query = "DELETE FROM recDetails where dtl_RecID=" . $recID;
if (!$mysqli->query($query)) {
$syserror = $mysqli->error;
print "<div style='color:red'> Error: Cannot delete record details " . $syserror . "</div>";
return null;
}
}
$ref = null;
//add-update Heurist record
$out = saveRecord($recID, $rectype, null, null, get_group_ids(), null, null, null, null, null, null, $details, null, null, null, null, null, $ref, $ref, 2);
if (@$out['error']) {
print "<br>Source record# " . $faims_id . " ";
print "=><div style='color:red'> Error: " . implode("; ", $out["error"]) . "</div>---<br>";
} else {
if ($recID) {
$cntUpdated++;
print "UPDATED as #" . $recID . "<br/>";
} else {
$cntInsterted++;
print "INSERTED as #" . $out["bibID"] . "<br/>";
$recID = $out["bibID"];
}
}
return $recID;
}
/**
* saves record in heurist DB
*
* @param mixed $recId
* @param mixed $recordType
* @param mixed $rec_URL
* @param mixed $details
* @param mixed $zotero_itemid
*/
function addRecordFromZotero($recId, $recordType, $rec_URL, $details, $zotero_itemid, $is_echo)
{
global $rep_errors_only, $dt_SourceRecordID;
$new_recid = null;
if (count($details) > 0) {
if ($zotero_itemid) {
$details["t:" . $dt_SourceRecordID] = array("0" => $zotero_itemid);
}
// 8) save rtecord
$ref = null;
if ($recId) {
//sice we do not know dtl_ID - remove all details for updated record
$query = "DELETE FROM recDetails where dtl_RecID=" . $recId;
$res = mysql_query($query);
if (!$res) {
$syserror = mysql_error();
print "<div style='color:red'> Error: Cannot delete record details " . $syserror . "</div>";
return;
}
}
//add-update Heurist record
$out = saveRecord($recId, $recordType, $rec_URL, null, null, null, null, null, null, null, null, $details, null, null, null, null, null, $ref, $ref, 2);
if (@$out['error']) {
print "<div style='color:red'> Error: " . implode("; ", $out["error"]) . "</div>";
} else {
$new_recid = $out["bibID"];
if ($is_echo) {
print '[' . ($recId ? "Updated" : "Added") . " Id " . $out["bibID"] . ']<br>';
}
if (!$rep_errors_only) {
if (@$out['warning']) {
print "<div style='color:red'>Warning: " . implode("; ", $out["warning"]) . "</div>";
}
}
}
}
return $new_recid;
}
function doTransfer()
{
global $sourcedbname, $dbPrefix, $db_prefix, $is_h2, $user_id_insource, $user_workgroups;
$sourcedb = $db_prefix . $sourcedbname;
$rep_errors_only = @$_REQUEST['reportlevel'] == "1";
echo "<p>Now copying data from <b>{$sourcedb}</b> to <b>" . $dbPrefix . HEURIST_DBNAME . "</b><p>Processing: ";
$terms_h2 = array();
$user_rights = null;
// Loop through types for all records in the database (actual records, not defined types)
if ($is_h2) {
//load all terms
$query1 = "SELECT `rdl_id`,`rdl_value` FROM `{$sourcedb}`.`rec_detail_lookups`";
$res1 = mysql_query($query1);
while ($row1 = mysql_fetch_array($res1)) {
$terms_h2[$row1[1]] = $row1[0];
}
$query1 = "SELECT DISTINCT (`rec_type`) FROM `{$sourcedb}`.`records`";
$user_rights = "rec_temporary=0";
} else {
$query1 = "SELECT DISTINCT (`rec_RecTypeID`) FROM {$sourcedb}.Records";
$user_rights = ' (rec_FlagTemporary=0 and (rec_OwnerUGrpID=' . $user_id_insource . ' or (not rec_NonOwnerVisibility="hidden")';
// rec_NonOwnerVisibility="public")
if (!empty($user_workgroups)) {
$user_rights = $user_rights . ' or rec_OwnerUGrpID in (' . join(',', $user_workgroups) . ')))';
} else {
$user_rights = $user_rights . '))';
}
}
if ($user_rights) {
$query1 = $query1 . " where " . $user_rights;
}
$res1 = mysql_query($query1);
if (!$res1) {
print "<br>Bad query for record type loop {$res1} <br>";
print "{$query1}<br>";
die("<p>Sorry ...</p>");
}
$added_records = array();
$unresolved_pointers = array();
$missed_terms = array();
$missed_terms2 = array();
/*$detailTypes = getAllDetailTypeStructures(); //in current database
$detailTypes = $detailTypes['typedefs'];
$fld_ind = $detailTypes['fieldNamesToIndex']['dty_Type']
$detailTypes[$dttype][$fld_ind];*/
print "<br>************************************************<br>Import records";
print "<br>The following section adds records and allocates them new IDs.";
if (!$rep_errors_only) {
print "<br>It reports this in the form Old ID => New ID";
}
// loop through the set of rectypes actually in the records in the database
while ($row1 = mysql_fetch_array($res1)) {
$rt = $row1[0];
if (!array_key_exists('cbr' . $rt, $_REQUEST)) {
continue;
}
$recordType = $_REQUEST['cbr' . $rt];
if (intval($recordType) < 1) {
print "<br>Record type {$rt} is not mapped";
ob_flush();
flush();
// flush to screen
continue;
}
//@todo - add record type name
$rt_counter = 0;
print "<br>Record type: {$rt}";
// tell user somethign is happening
ob_flush();
flush();
// flush to screen
if ($is_h2) {
$query2 = "select `rec_id`,`rec_url` from `{$sourcedb}`.`records` Where `{$sourcedb}`.`records`.`rec_type`={$rt}";
} else {
$query2 = "select `rec_ID`,`rec_URL` from {$sourcedb}.Records Where {$sourcedb}.Records.rec_RecTypeID={$rt}";
}
if ($user_rights) {
$query2 = $query2 . " and " . $user_rights;
}
$res2 = mysql_query($query2);
if (!$res2) {
print "<div style='color:red;'>Bad query for records loop for source record type {$rt}</div>";
print "<br>Query: {$query2}";
ob_flush();
flush();
// flush to screen
continue;
//die ("<p>Sorry ...</p>");
}
//special detailtype to keep original record id
$dt_SourceRecordID = defined('DT_ORIGINAL_RECORD_ID') ? DT_ORIGINAL_RECORD_ID : 0;
while ($row2 = mysql_fetch_array($res2)) {
//select details and create details array
$rid = $row2[0];
//record id
// print "<br>".$rid." ";
if ($is_h2) {
$query3 = "SELECT `rd_type`, `rdt_type`, `rd_val`, `rd_file_id`, astext(`rd_geo`)\n\t\t\t\t\t\t\tFROM `{$sourcedb}`.`rec_details` rd, `{$sourcedb}`.`rec_detail_types` dt where rd.`rd_type`=dt.`rdt_id` and rd.`rd_rec_id`={$rid} order by `rd_type`";
} else {
$query3 = "SELECT `dtl_DetailTypeID`, `dty_Type`, `dtl_Value`, `dtl_UploadedFileID`, astext(`dtl_Geo`)\n\t\t\t\t\t\t\tFROM {$sourcedb}.`recDetails` rd, {$sourcedb}.`defDetailTypes` dt where rd.`dtl_DetailTypeID`=dt.`dty_ID` and rd.`dtl_RecID`={$rid} order by `dtl_DetailTypeID`";
}
$res3 = mysql_query($query3);
// todo: check query was successful
if (!$res3) {
print "<br>record " . $rid . " <div style='color:red;'>bad select of detail fields</div>";
print "<br>query: {$query3}";
ob_flush();
flush();
// flush to screen
continue;
//die ("<p>Sorry ...</p>");
}
$unresolved_records = array();
$details = array();
$dtid = 0;
$key = 0;
$ind = 0;
$values = array();
//add special detail type 2-589 - reference to original record id
if ($dt_SourceRecordID > 0) {
$details["t:" . $dt_SourceRecordID] = array('0' => $rid);
}
while ($row3 = mysql_fetch_array($res3)) {
if ($dtid != $row3[0]) {
if ($key > 0) {
$details["t:" . $key] = $values;
}
$dtid = $row3[0];
$values = array();
$ind;
}
if (!array_key_exists('cbd' . $row3[0], $_REQUEST)) {
continue;
}
$key = $_REQUEST['cbd' . $row3[0]];
if (intval($key) < 1) {
if ($rt == 52) {
//debug
print "mapping not defined for detail (field) #" . $dtid;
}
//mapping for this detail type is not specified
continue;
}
$value = $row3[2];
//determine the type of filedtype
if ($row3[1] == 'enum' || $row3[1] == 'relationtype') {
if ($is_h2) {
if (array_key_exists($value, $terms_h2)) {
$value = $terms_h2[$value];
} else {
if (array_search($value, $missed_terms) == false) {
array_push($missed_terms, $value);
}
}
}
$term = getDestinationTerm($value);
if ($term == null) {
$ind = array_search(intval($value), $missed_terms2);
if (count($missed_terms2) == 0 || $ind == 0 && $missed_terms2[$ind] != intval($value)) {
array_push($missed_terms2, intval($value));
}
$value = null;
} else {
$value = $term;
}
} else {
if ($row3[1] == 'file') {
if ($is_h2) {
$value = copyRemoteFileH2($row3[3]);
//returns new file id
} else {
$value = copyRemoteFile($row3[3]);
//returns new file id
}
} else {
if ($row3[1] == 'geo') {
$value = $value . " " . $row3[4];
// string geotype+space+wkt
} else {
if ($row3[1] == 'relmarker') {
} else {
if ($row3[1] == 'resource') {
//find the id of record in destionation database among pairs of added records
if (array_key_exists($value, $added_records)) {
$value = $added_records[$value];
} else {
array_push($unresolved_records, $key . "|" . $value);
//print "<div style='color:#ffaaaa;'>resource record#".$value." not found</div>";
$value = null;
//ingnore
}
}
}
}
}
}
if ($value != null) {
$values[$ind] = $value;
$ind++;
}
}
//while by details
//for last one
if ($key > 0 && count($values) > 0) {
$details["t:" . $key] = $values;
}
/*****DEBUG****/
//error_log("DETAILS:>>>>".print_r($details,true));
$ref = null;
//add-update Heurist record
$out = saveRecord(null, $recordType, $row2[1], null, null, null, null, null, null, null, null, $details, null, null, null, null, null, $ref, $ref, 2);
if (@$out['error']) {
print "<br>Source record# " . $rid . " ";
print "=><div style='color:red'> Error: " . implode("; ", $out["error"]) . "</div>";
} else {
$new_recid = $out["bibID"];
$added_records[$rid] = $new_recid;
$rt_counter++;
if (count($unresolved_records) > 0) {
$unresolved_pointers[$new_recid] = $unresolved_records;
}
if (!$rep_errors_only) {
print "<br>" . $rid . " => " . $out["bibID"];
if (@$out['warning']) {
print "<br>Warning: " . implode("; ", $out["warning"]);
}
}
}
}
//while by record of particular record type
if ($rt_counter > 0) {
print " => added {$rt_counter} records";
}
ob_flush();
flush();
// flush to screen
}
// end of loop for record types
/*****DEBUG****/
//error_log("DEBUG: UNRESOLVED POINTERS>>>>>".print_r($unresolved_pointers, true));
if (count($missed_terms) > 0) {
print "<br><br>*********************************************************";
print "<br>These terms IDs are not found in {$sourcedb}<br>";
print implode('<br>', $missed_terms);
}
if (count($missed_terms2) > 0) {
print "<br><br>*********************************************************";
print "<br>Mapping for these terms IDs is not specified<br>";
print implode('<br>', $missed_terms2);
}
if (count($unresolved_pointers) > 0) {
$notfound_rec = array();
print "<br><br>*********************************************************";
print "<br>Finding and setting unresolved record pointers<br>";
if (!$rep_errors_only) {
print "<br>It reports in form: source RecID => now target pointer RecID => in Rec Id<br>";
}
//resolve record pointers
$inserts = array();
foreach ($unresolved_pointers as $recid => $unresolved_records) {
foreach ($unresolved_records as $code) {
//print "<br>".$code;
$aa = explode("|", $code);
$dt_id = $aa[0];
$src_recid = $aa[1];
//print " ".$dt_id."=".$src_recid;
if (array_key_exists($src_recid, $added_records)) {
if (!$rep_errors_only) {
print "<br>" . $src_recid . "=>" . $added_records[$src_recid] . "=>" . $recid;
}
array_push($inserts, "({$recid}, {$dt_id}, " . $added_records[$src_recid] . ", 1)");
} else {
if (array_search($src_recid, $notfound_rec) == false) {
array_push($notfound_rec, $src_recid . " for " . $recid);
}
}
}
}
if (count($notfound_rec) > 0) {
print "<br>These records are specified as pointers in source database but they were not added into target database:<br>";
print implode('<br>', $notfound_rec);
}
if (count($inserts) > 0) {
//insert all new details
$query1 = "insert into {$dbPrefix}" . HEURIST_DBNAME . ".recDetails (dtl_RecID, dtl_DetailTypeID, dtl_Value, dtl_AddedByImport) values " . join(",", $inserts);
/*****DEBUG****/
//error_log(">>>>>>>>>>>>>>>".$query1);
mysql_query($query1);
print "<br><br>Total count of resolved pointers:" . count($inserts);
}
}
print "<br><br><br><h3>Transfer completed - <a href=../../index.php?db=" . HEURIST_DBNAME . " title='Return to the main search page of the current database'><b>return to main page</b></a></h3>";
}
$loggedin = 0;
break;
case 'delete':
deleteRecord($idnum);
break;
case 'edit':
//nothing here
break;
case 'cancel':
//nothing here
break;
case 'save':
if ($idnum == -1) {
addRecord($name, $notes, $enddate, $person);
} else {
saveRecord($idnum, $name, $notes, $enddate, $person);
}
break;
case 'add':
//nothing here
break;
}
################################################################
## HTML
################################################################
$table = BuildTable();
if ($msg == '') {
#$msg = "SESSION: " . $_SESSION['loggedin'];
#. ">>>>" . (mktime() - $_SESSION['time']);
}
include 'include/html.php';
}
mysqli_stmt_close($stmt);
} else {
//prepare failed
echo "WRONG2";
}
}
?>
<html>
<head>
<!-- I USE BOOTSTRAP BECAUSE IT MAKES FORMATTING/LIFE EASIER -->
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css"><!-- Latest compiled and minified CSS -->
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap-theme.min.css"><!-- Optional theme -->
<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script><!-- Latest compiled and minified JavaScript -->
</head>
<body>
<div class="container">
<form action = "insert2.php" method="POST">
<?php
displayOptionOfCity($result);
if (isset($_POST['save'])) {
saveRecord();
}
?>
</form>
</div>
</body>
</html>
<?php
mysqli_close($link);
function doTransfer()
{
global $sourcedbname, $dbPrefix, $db_prefix, $is_h2, $user_id_insource, $user_workgroups;
$sourcedb = $db_prefix . $sourcedbname;
$rep_errors_only = @$_REQUEST['reportlevel'] == "1";
echo "<p>Now copying data from <b>{$sourcedb}</b> to <b>" . $dbPrefix . HEURIST_DBNAME . "</b><p>Processing: ";
$terms_h2 = array();
$user_rights = null;
// Loop through types for all records in the database (actual records, not defined types)
if ($is_h2) {
//load all terms
$query1 = "SELECT `rdl_id`,`rdl_value` FROM `{$sourcedb}`.`rec_detail_lookups`";
$res1 = mysql_query($query1);
while ($row1 = mysql_fetch_array($res1)) {
$terms_h2[$row1[1]] = $row1[0];
}
$query1 = "SELECT DISTINCT (`rec_type`) FROM `{$sourcedb}`.`records`";
$user_rights = "rec_temporary=0";
} else {
$query1 = "SELECT DISTINCT (`rec_RecTypeID`) FROM {$sourcedb}.Records";
$user_rights = ' (rec_FlagTemporary=0 and (rec_OwnerUGrpID=' . $user_id_insource . ' or (not rec_NonOwnerVisibility="hidden")';
// rec_NonOwnerVisibility="public")
if (!empty($user_workgroups)) {
$user_rights = $user_rights . ' or rec_OwnerUGrpID in (' . join(',', $user_workgroups) . ')))';
} else {
$user_rights = $user_rights . '))';
}
}
if ($user_rights) {
$query1 = $query1 . " where " . $user_rights;
}
$res1 = mysql_query($query1);
if (!$res1) {
print "<br>Bad query for record type loop {$res1} <br>";
print "{$query1}<br>";
die("<p>Sorry ...</p>");
}
$added_records = array();
$unresolved_pointers = array();
$missed_terms = array();
$missed_terms2 = array();
/*$detailTypes = getAllDetailTypeStructures(); //in current database
$detailTypes = $detailTypes['typedefs'];
$fld_ind = $detailTypes['fieldNamesToIndex']['dty_Type']
$detailTypes[$dttype][$fld_ind];*/
print "<br>************************************************<br>Import records";
print "<br>The following section adds records and allocates them new IDs.";
if (!$rep_errors_only) {
print "<br>It reports this in the form Old ID => New ID";
}
// loop through the set of rectypes actually in the records in the database
while ($row1 = mysql_fetch_array($res1)) {
$rt = $row1[0];
if (!array_key_exists('cbr' . $rt, $_REQUEST)) {
continue;
}
$recordType = $_REQUEST['cbr' . $rt];
if (intval($recordType) < 1) {
print "<br><b>Record type {$rt} is not mapped</b>";
ob_flush();
flush();
// flush to screen
continue;
}
//@todo - add record type name
$rt_counter = 0;
print "<br><br><br><b>Record type: {$rt}</b>";
// tell user something is happening
ob_flush();
flush();
// flush to screen
if ($is_h2) {
// version 2 database
$query2 = "select `rec_id`,`rec_url` from `{$sourcedb}`.`records` Where `{$sourcedb}`.`records`.`rec_type`={$rt}";
} else {
// vedrsion 3 or later
$query2 = "select `rec_ID`,`rec_URL` from {$sourcedb}.Records Where {$sourcedb}.Records.rec_RecTypeID={$rt}";
}
if ($user_rights) {
$query2 = $query2 . " and " . $user_rights;
}
$res2 = mysql_query($query2);
if (!$res2) {
print "<div style='color:red;'><b>Bad query for records loop for source record type {$rt}</b></div>";
print "<br>Query: {$query2}";
ob_flush();
flush();
// flush to screen
continue;
//die ("<p>Sorry ...</p>");
}
//special detailtype to keep original record id
$dt_SourceRecordID = defined('DT_ORIGINAL_RECORD_ID') ? DT_ORIGINAL_RECORD_ID : 0;
while ($row2 = mysql_fetch_array($res2)) {
//select details and create details array
$rid = $row2[0];
//record id
if ($is_h2) {
// Heurist version 2 database
$query3 = "SELECT `rd_type`, `rdt_type`, `rd_val`, `rd_file_id`, AsWKT(`rd_geo`)\n FROM `{$sourcedb}`.`rec_details` rd, `{$sourcedb}`.`rec_detail_types` dt where rd.`rd_type`=dt.`rdt_id` and rd.`rd_rec_id`={$rid} order by `rd_type`";
} else {
// version 3 or later
$query3 = "SELECT `dtl_DetailTypeID`, `dty_Type`, `dtl_Value`, `dtl_UploadedFileID`, AsWKT(`dtl_Geo`)\n FROM {$sourcedb}.`recDetails` rd, {$sourcedb}.`defDetailTypes` dt where rd.`dtl_DetailTypeID`=dt.`dty_ID` and rd.`dtl_RecID`={$rid} order by `dtl_DetailTypeID`";
}
$res3 = mysql_query($query3);
// todo: check query was successful
if (!$res3) {
print "<br>record " . $rid . " <div style='color:red;'><b>bad select query for detail fields, data for this record may not be correct</b></div>";
print "<br>query: {$query3}";
ob_flush();
flush();
// flush to screen
continue;
}
$unresolved_records = array();
$details = array();
$dtid = 0;
$key = 0;
$ind = 0;
$values = array();
//add special detail type 2-589 - reference to original record id
if ($dt_SourceRecordID > 0) {
$details["t:" . $dt_SourceRecordID] = array('0' => $rid);
}
while ($row3 = mysql_fetch_array($res3)) {
if ($dtid != $row3[0]) {
if ($key > 0) {
$details["t:" . $key] = $values;
}
$dtid = $row3[0];
$values = array();
$ind;
}
if (!array_key_exists('cbd' . $row3[0], $_REQUEST)) {
continue;
}
$key = $_REQUEST['cbd' . $row3[0]];
if (intval($key) < 1) {
if ($rt == 52) {
//debug TODO: What the hell is this? What is rt 52?
print "mapping not defined for detail (field) #" . $dtid;
}
//mapping for this detail type is not specified
continue;
}
$value = $row3[2];
//determine the type of field type
if ($row3[1] == 'enum' || $row3[1] == 'relationtype') {
if ($is_h2) {
if (array_key_exists($value, $terms_h2)) {
$value = $terms_h2[$value];
} else {
if (array_search($value, $missed_terms) == false) {
array_push($missed_terms, $value);
}
}
}
$term = getDestinationTerm($value);
if ($term == null) {
$ind = array_search(intval($value), $missed_terms2);
if (count($missed_terms2) == 0 || $ind == 0 && $missed_terms2[$ind] != intval($value)) {
array_push($missed_terms2, intval($value));
}
$value = null;
} else {
$value = $term;
}
} else {
if ($row3[1] == 'file') {
if ($is_h2) {
$value = copyRemoteFileH2($row3[3]);
//returns new file id
} else {
$value = copyRemoteFile($row3[3]);
//returns new file id
}
} else {
if ($row3[1] == 'geo') {
$value = $value . " " . $row3[4];
// string geotype+space+wkt
} else {
if ($row3[1] == 'relmarker') {
// no data stored for relationship markers
} else {
if ($row3[1] == 'resource') {
// record pointer field
// find the id of record in destination database among pairs of added records
if (array_key_exists($value, $added_records)) {
$value = $added_records[$value];
} else {
array_push($unresolved_records, $key . "|" . $value);
//print "<div style='color:#ffaaaa;'>resource record#".$value." not found</div>";
$value = null;
//ingnore
}
}
}
}
}
}
if ($value != null) {
$values[$ind] = $value;
$ind++;
}
}
//while by details
//for last one
if ($key > 0 && count($values) > 0) {
$details["t:" . $key] = $values;
}
$ref = null;
//add-update Heurist record
$out = saveRecord(null, $recordType, $row2[1], null, null, null, null, null, null, null, null, $details, null, null, null, null, null, $ref, $ref, 2);
if (@$out['error']) {
print "<br>Source record# " . $rid . " ";
print "=><div style='color:red'> Error: " . implode("; ", $out["error"]) . "</div>";
} else {
$new_recid = $out["bibID"];
$added_records[$rid] = $new_recid;
$rt_counter++;
if (count($unresolved_records) > 0) {
$unresolved_pointers[$new_recid] = $unresolved_records;
}
if (!$rep_errors_only) {
// in active writing mode
if (@$out['warning']) {
print "<br>Warning: " . implode("; ", $out["warning"]);
}
print "<br>saved: rec#" . $rid . " => rec#" . $out["bibID"];
}
}
}
//while by record of particular record type
if ($rt_counter > 0) {
print " => added {$rt_counter} records<br>";
}
ob_flush();
flush();
// flush to screen
}
// end of loop for record types
if (count($missed_terms) > 0) {
print "<br><br>*********************************************************";
print "<br>These terms IDs are not found in {$sourcedb}<br>";
print implode('<br>', $missed_terms);
}
if (count($missed_terms2) > 0) {
print "<br><br>*********************************************************";
print "<br>Mapping for these term IDs is not specified<br>";
print implode('<br>', $missed_terms2);
}
if (count($unresolved_pointers) > 0) {
$notfound_rec = array();
print "<br><br>*********************************************************";
print "<br>Finding and setting unresolved record pointers<br>";
if (!$rep_errors_only) {
print "<br>Report is in the form: source RecID => new target pointer RecID => in Rec Id<br>";
}
//resolve record pointers
$inserts = array();
foreach ($unresolved_pointers as $recid => $unresolved_records) {
foreach ($unresolved_records as $code) {
//print "<br>".$code;
$aa = explode("|", $code);
$dt_id = $aa[0];
$src_recid = $aa[1];
//print " ".$dt_id."=".$src_recid;
if (array_key_exists($src_recid, $added_records)) {
if (!$rep_errors_only) {
print "<br>" . $src_recid . "=>" . $added_records[$src_recid] . "=>" . $recid;
}
array_push($inserts, "({$recid}, {$dt_id}, " . $added_records[$src_recid] . ", 1)");
} else {
if (array_search($src_recid, $notfound_rec) == false) {
array_push($notfound_rec, $src_recid . " for " . $recid);
}
}
}
}
if (count($notfound_rec) > 0) {
print "<br>These records are specified as pointers in source database but they were not added into target database:<br>";
print implode('<br>', $notfound_rec);
}
if (count($inserts) > 0) {
//insert all new details
$query1 = "insert into {$dbPrefix}" . HEURIST_DBNAME . ".recDetails (dtl_RecID, dtl_DetailTypeID, dtl_Value, dtl_AddedByImport) values " . join(",", $inserts);
mysql_query($query1);
print "<br><br>Total count of resolved pointers:" . count($inserts);
}
}
//IMPORT TAGS/KEYWORDS ==================================
// get all target users
// find corresspondant source users
// find all tags with records for source user
// add tags to target tag table
// add tag links in target for all imported records
// get all target users
$allusers = getUsers();
//in target database
$rkeys = array_keys($_REQUEST);
$tags_corr = array();
foreach ($rkeys as $rkey) {
if (strpos($rkey, 'cbu') === 0 && @$allusers[$_REQUEST[$rkey]]) {
// find corresspondant source users
$source_uid = substr($rkey, 3);
$target_uid = $_REQUEST[$rkey];
//find all tags with records for source user
if ($is_h2) {
$query1 = "select kwd.kwd_id, kwd.kwd_name, count(kwl_rec_id)\n from `{$sourcedb}`.`keywords` kwd, `{$sourcedb}`.`keyword_links` kwl\n where kwd.kwd_usr_id={$source_uid} and kwl.kwl_kwd_id=kwd.kwd_id\n group by kwd.kwd_id\n order by kwd.kwd_id";
} else {
$query1 = "select kwd.tag_ID, kwd.tag_Text, count(rtl_RecID)\n from `{$sourcedb}`.`usrTags` kwd, `{$sourcedb}`.`usrRecTagLinks` kwl\n where kwd.tag_UGrpID={$source_uid} and kwl.rtl_TagID=kwd.tag_ID\n group by kwd.tag_ID\n order by kwd.tag_ID";
}
//add tags to target tag table
$res1 = mysql_query($query1);
while ($row1 = mysql_fetch_array($res1)) {
$ress = get_ids_for_tags(array($row1[1]), true, $target_uid);
if (count($ress) > 0) {
$tags_corr[$row1[0]] = array($ress[0], $row1[1], 0, $target_uid);
}
}
}
}
//foreach users
foreach ($added_records as $oldid => $newid) {
// find tag links in source database
if ($is_h2) {
$query1 = "select kwl.kwl_kwd_id\n from `{$sourcedb}`.`keyword_links` kwl\n where kwl.kwl_rec_id=" . $oldid;
} else {
$query1 = "select kwl.rtl_TagID\n from `{$sourcedb}`.`usrRecTagLinks` kwl\n where kwl.rtl_RecID=" . $oldid;
}
$res1 = mysql_query($query1);
while ($row1 = mysql_fetch_array($res1)) {
$newtag = @$tags_corr[$row1[0]];
if ($newtag) {
mysql_query('insert ignore into `' . USERS_DATABASE . '`.usrRecTagLinks (rtl_RecID, rtl_TagID, rtl_AddedByImport) ' . ' values (' . $newid . ',' . $newtag[0] . ',1)');
$tag_count = mysql_affected_rows();
if (mysql_error()) {
} else {
if ($tag_count > 0) {
$tags_corr[$row1[0]][2]++;
mysql_query('insert into `' . USERS_DATABASE . '`.usrBookmarks (bkm_UGrpID, bkm_Added, bkm_Modified, bkm_recID, bkm_AddedByImport) ' . 'values( ' . $newtag[3] . ', now(), now(), ' . $newid . ',1 )');
}
}
}
}
}
//report
print "<br />*********************************************************<br />";
print "Tags added<br /><br />";
foreach ($tags_corr as $oldid => $newtag) {
print $oldid . " " . $newtag[1] . " ==> " . $newtag[0] . ", " . $newtag[2] . " records<br />";
}
print "<br /><br />";
//FINAL ==================================
print "<br><br><br><h3>Transfer completed - <a href=getRecordsFromDB.php?db=" . HEURIST_DBNAME . " title='Return to the main search page of the current database'><b>return to main page</b></a></h3>";
}
/**
*
* @global type $rep_counter
* @global string $rep_issues
* @global array $fieldhelper_to_heurist_map
* @param type $dir
*/
function doHarvestInDir($dir)
{
global $rep_issues, $fieldhelper_to_heurist_map, $mediaExts, $progress_divid, $geoDT, $fileDT, $titleDT, $startdateDT, $enddateDT, $descriptionDT;
$rep_processed = 0;
$rep_added = 0;
$rep_updated = 0;
$rep_processed_dir = 0;
$rep_ignored = 0;
$f_items = null;
//reference to items element
$progress_divid++;
ob_start();
print "<div><b>{$dir}</b><span id='progress{$progress_divid}'></span></div>";
ob_flush();
flush();
if (!is_writable($dir)) {
print "<div style=\"color:red\">Folder is not writeable. Check permissions</div>";
return 0;
}
$manifest_file = $dir . "fieldhelper.xml";
//list of all files in given folder - need to treat new files that are not mentioned in manifest file
$all_files = scandir($dir);
if (file_exists($manifest_file)) {
//read fieldhelper.xml
if (is_readable($manifest_file)) {
//check write permission
if (!is_writable($manifest_file)) {
print "<div style=\"color:red\">Manifest is not writeable. Check permissions.</div>";
return 0;
}
} else {
print "<div style=\"color:red\">Manifest is not readable. Check permissions.</div>";
return 0;
}
$fh_data = simplexml_load_file($manifest_file);
if ($fh_data == null || is_string($fh_data)) {
print "<div style=\"color:red\">Manifest is corrupted</div>";
return 0;
}
//MAIN LOOP in manifest
$not_found = true;
//true if manifest is empty
foreach ($fh_data->children() as $f_gen) {
if ($f_gen->getName() == "items") {
$f_items = $f_gen;
$not_found = false;
$tot_files = count($f_gen->children());
$cnt_files = 0;
foreach ($f_gen->children() as $f_item) {
$recordId = null;
$recordType = RT_MEDIA_RECORD;
//media by default
$recordURL = null;
$recordNotes = null;
$el_heuristid = null;
$lat = null;
$lon = null;
$filename = null;
$filename_base = null;
//filename only
$details = array();
$file_id = null;
$old_md5 = null;
foreach ($f_item->children() as $el) {
//$key=>$value
$content = strval($el);
// (string)$el;
$key = $el->getName();
$value = $content;
if ($key == "md5") {
$old_md5 = $value;
} else {
if (@$fieldhelper_to_heurist_map[$key]) {
$key2 = $fieldhelper_to_heurist_map[$key];
if ($key2 == "file") {
$filename = $dir . $value;
$filename_base = $value;
$key3 = $fieldhelper_to_heurist_map['file_name'];
if ($key3 > 0) {
$details["t:" . $key3] = array("1" => $value);
}
$key3 = $fieldhelper_to_heurist_map['file_path'];
if ($key3 > 0) {
$relative_path = getRelativePath(HEURIST_FILESTORE_DIR, $dir);
$details["t:" . $key3] = array("1" => $relative_path);
//change to relative path
}
} else {
if ($key2 == "lat") {
$lat = floatval($value);
} else {
if ($key2 == "lon") {
$lon = floatval($value);
} else {
if ($key2 == "recordId") {
$recordId = $value;
$el_heuristid = $el;
} else {
if (intval($key2) > 0) {
//add to details
$details["t:" . $key2] = array("1" => $value);
}
}
}
}
}
// else field type not defined in this instance
}
}
}
//for item keys
if ($filename) {
//exclude from the list of all files in this folder
if (in_array($filename_base, $all_files)) {
$ind = array_search($filename_base, $all_files, true);
unset($all_files[$ind]);
}
}
if ($recordId != null) {
//veify that this record exists
$res = mysql__select_array("Records", "rec_ID", "rec_ID=" . $recordId);
if (!(is_array($res) && count($res) > 0)) {
print "<div>File: <i>{$filename_base}</i> was indexed as rec# {$recordId}. " . "This record was not found. File will be reindexed</div>";
$recordId = null;
}
}
if ($recordId == null) {
//import only new
if ($filename) {
if (file_exists($filename)) {
$currfile = $filename;
//assign to global
//add-update the uploaded file
$file_id = register_file($filename, null, false);
if (is_numeric($file_id)) {
$details["t:" . $fileDT] = array("1" => $file_id);
//read EXIF data for JPEG images
$recordNotes = readEXIF($filename);
} else {
print "<div>File: <i>{$filename_base}</i> <span style=\"color:red\">" . "Error: Failed to register. No record created</span></div>";
$file_id = null;
}
} else {
print "<div>File: <i>{$filename_base}</i> <span style=\"color:red\">" . "File is referenced in fieldhelper.xml but was not found on disk." . "No record was created.</span></div>";
}
}
if (!$file_id) {
continue;
//add with valid file only
}
if (is_numeric($lat) && is_numeric($lon) && ($lat != 0 || $lon != 0)) {
$details["t:" . $geoDT] = array("1" => "p POINT ({$lon} {$lat})");
}
//set title by default
if (!array_key_exists("t:" . $titleDT, $details)) {
$details["t:" . $titleDT] = array("1" => $filename);
print "<div>File: <i>{$filename_base}</i> <span style=\"color:#ff8844\">" . "Warning: there was no title recorded in the XML manifest for this file." . "Using file path + file name as title.</span></div>";
}
$new_md5 = null;
$key = $fieldhelper_to_heurist_map['md5'];
if ($key > 0) {
$new_md5 = md5_file($filename);
$details["t:" . $key] = array("1" => $new_md5);
}
$out['error'] = 'test';
//add-update Heurist record
$out = saveRecord($recordId, $recordType, $recordURL, $recordNotes, null, null, null, null, null, null, null, $details, null, null, null, null, null);
if (@$out['error']) {
print "<div>File: <i>{$filename_base}</i> <span style='color:red'>Error: " . implode("; ", $out["error"]) . "</span></div>";
} else {
if ($new_md5 == null) {
$new_md5 = md5_file($filename);
}
//update xml
if ($recordId == null) {
if ($old_md5 != $new_md5) {
print "<div>File: <i>{$filename_base}</i> <span style=\"color:#ff8844\">" . "Warning: Checksum differs from value in manifest; " . "data file may have been changed</span></div>";
}
$f_item->addChild("heurist_id", $out["bibID"]);
$f_item->addChild("md5", $new_md5);
$f_item->addChild("filesize", filesize($filename));
$rep_added++;
} else {
$el_heuristid["heurist_id"] = $out["bibID"];
$rep_updated++;
}
if (@$out['warning']) {
print "<div>File: <i>{$filename_base}</i> <span style=\"color:#ff8844\">Warning: " . implode("; ", $out["warning"]) . "</span></div>";
}
}
$rep_processed++;
} else {
$rep_ignored++;
}
$cnt_files++;
if ($cnt_files % 5 == 0) {
ob_start();
print '<script type="text/javascript">update_counts(' . $progress_divid . ',' . $cnt_files . ',' . $rep_processed . ',' . $tot_files . ')</script>' . "\n";
ob_flush();
flush();
}
}
//for items
}
//if has items
}
//for all children in manifest
if ($not_found) {
print "<div style=\"color:red\">Manifest is either corrupted or empty</div>";
} else {
if ($rep_processed > 0) {
print "<div>{$rep_processed} files processed. {$rep_added} added. {$rep_updated} updated.</div>";
}
if ($rep_ignored > 0) {
print "<div>{$rep_ignored} files already indexed.</div>";
}
}
} else {
//create empty manifest XML - TODO!!!!
$s_manifest = <<<XML
<?xml version="1.0" encoding="UTF-8"?>
<fieldhelper version="1">
<info>
<AppName>Heurist</AppName>
<AppVersion>v 3.0.0 2012-01-01</AppVersion>
<AppCopyright>© ArtEresearch, University of Sydney</AppCopyright>
<date></date>
</info>
<formatOutput>1</formatOutput>
</fieldhelper>
XML;
$fh_data = simplexml_load_string($s_manifest);
}
// add new empty items element
if ($f_items == null) {
$f_items = $fh_data->addChild("items");
}
$tot_files = count($all_files);
$cnt_files = 0;
$cnt_added = 0;
//for files in folder that are not specified in the manifest file
foreach ($all_files as $filename) {
if (!($filename == "." || $filename == ".." || is_dir($dir . $filename) || $filename == "fieldhelper.xml")) {
$filename_base = $filename;
$filename = $dir . $filename;
$currfile = $filename;
$flleinfo = pathinfo($filename);
$recordNotes = null;
//checks for allowed extensions
if (in_array(strtolower(@$flleinfo['extension']), $mediaExts)) {
$details = array();
$file_id = register_file($filename, null, false);
if (is_numeric($file_id)) {
$details["t:" . $fileDT] = array("1" => $file_id);
//read EXIF data for JPEG images
$recordNotes = readEXIF($filename);
} else {
print "<div>File: <i>{$filename_base}</i> <span style=\"color:#ff8844\">" . "Warning: failed to register. No record created for: .{$file_id}</span></div>";
//$rep_issues = $rep_issues."<br/>Can't register file:".$filename.". ".$file_id;
$file_id = null;
continue;
}
$details["t:" . $titleDT] = array("1" => $flleinfo['basename']);
/* TODO - extract these data from exif
$details["t:".$descriptionDT] = array("1"=>$file_id);
$details["t:".$startdateDT] = array("1"=>$file_id);
$details["t:".$enddateDT] = array("1"=>$file_id);
$details["t:".$geoDT] = array("1"=>$file_id);
*/
$new_md5 = md5_file($filename);
$key = $fieldhelper_to_heurist_map['md5'];
if ($key > 0) {
$details["t:" . $key] = array("1" => $new_md5);
}
$key = $fieldhelper_to_heurist_map['file_name'];
if ($key > 0) {
$details["t:" . $key] = array("1" => $flleinfo['basename']);
}
$key = $fieldhelper_to_heurist_map['file_path'];
if ($key > 0) {
$targetPath = $flleinfo['dirname'];
$rel_path = getRelativePath(HEURIST_FILESTORE_DIR, $targetPath);
//getRelativePath2($targetPath);
$details["t:" . $key] = array("1" => $rel_path);
/*print "<div>".HEURIST_FILESTORE_DIR."</div>";
print "<div>file path :".$targetPath."</div>";
print "<div>relative path :".strpos($targetPath, HEURIST_FILESTORE_DIR)."--".$rel_path."</div>";
print "<div>relative path old :".getRelativePath(HEURIST_FILESTORE_DIR, $targetPath)."<br><br></div>";*/
}
$key = $fieldhelper_to_heurist_map['extension'];
if ($key > 0) {
$details["t:" . $key] = array("1" => $flleinfo['extension']);
}
$key = $fieldhelper_to_heurist_map['filesize'];
if ($key > 0) {
$details["t:" . $key] = array("1" => filesize($filename));
}
//add-update Heurist record
$out['error'] = 'test2';
$out = saveRecord(null, RT_MEDIA_RECORD, null, $recordNotes, null, null, null, null, null, null, null, $details, null, null, null, null, null);
$f_item = $f_items->addChild("item");
$f_item->addChild("filename", htmlspecialchars($flleinfo['basename']));
$f_item->addChild("nativePath", htmlspecialchars($filename));
$f_item->addChild("folder", htmlspecialchars($flleinfo['dirname']));
$f_item->addChild("extension", $flleinfo['extension']);
//$f_item->addChild("DateTime", );
//$f_item->addChild("DateTimeOriginal", );
$f_item->addChild("filedate", date("Y/m/d H:i:s.", filemtime($filename)));
$f_item->addChild("typeContent", "image");
$f_item->addChild("device", "image");
$f_item->addChild("duration", "2000");
$f_item->addChild("original_metadata", "chk");
$f_item->addChild("Name0", htmlspecialchars($flleinfo['basename']));
$f_item->addChild("md5", $new_md5);
$f_item->addChild("filesize", filesize($filename));
if (@$out['error']) {
print "<div>Fle: <i>{$filename_base}</i> <span style='color:red'>Error: " . implode("; ", $out["error"]) . "</span></div>";
} else {
$f_item->addChild("heurist_id", $out["bibID"]);
$cnt_added++;
}
$rep_processed_dir++;
}
//check ext
$cnt_files++;
if ($cnt_files % 5 == 0) {
ob_start();
print '<script type="text/javascript">update_counts(' . $progress_divid . ',' . $cnt_files . ',' . $cnt_added . ',' . $tot_files . ')</script>' . "\n";
ob_flush();
flush();
}
}
}
//for files in folder that are not specified in the directory
ob_start();
if ($rep_processed_dir > 0) {
print "<div style=\"color:green\">{$rep_processed_dir} processed. {$cnt_added} records created (new entries added to manifests)</div>";
}
print '<script type="text/javascript">update_counts(' . $progress_divid . ',' . $cnt_files . ',' . $cnt_added . ',0)</script>' . "\n";
ob_flush();
flush();
if ($rep_processed + $rep_processed_dir > 0) {
//save modified xml (with updated heurist_id tags)
$fh_data->formatOutput = true;
$fh_data->saveXML($manifest_file);
}
return $rep_processed + $rep_processed_dir;
}
