function doTransfer() { global $sourcedbname, $dbPrefix, $db_prefix, $is_h2, $user_id_insource, $user_workgroups; $sourcedb = $db_prefix . $sourcedbname; $rep_errors_only = @$_REQUEST['reportlevel'] == "1"; echo "<p>Now copying data from <b>{$sourcedb}</b> to <b>" . $dbPrefix . HEURIST_DBNAME . "</b><p>Processing: "; $terms_h2 = array(); $user_rights = null; // Loop through types for all records in the database (actual records, not defined types) if ($is_h2) { //load all terms $query1 = "SELECT `rdl_id`,`rdl_value` FROM `{$sourcedb}`.`rec_detail_lookups`"; $res1 = mysql_query($query1); while ($row1 = mysql_fetch_array($res1)) { $terms_h2[$row1[1]] = $row1[0]; } $query1 = "SELECT DISTINCT (`rec_type`) FROM `{$sourcedb}`.`records`"; $user_rights = "rec_temporary=0"; } else { $query1 = "SELECT DISTINCT (`rec_RecTypeID`) FROM {$sourcedb}.Records"; $user_rights = ' (rec_FlagTemporary=0 and (rec_OwnerUGrpID=' . $user_id_insource . ' or (not rec_NonOwnerVisibility="hidden")'; // rec_NonOwnerVisibility="public") if (!empty($user_workgroups)) { $user_rights = $user_rights . ' or rec_OwnerUGrpID in (' . join(',', $user_workgroups) . ')))'; } else { $user_rights = $user_rights . '))'; } } if ($user_rights) { $query1 = $query1 . " where " . $user_rights; } $res1 = mysql_query($query1); if (!$res1) { print "<br>Bad query for record type loop {$res1} <br>"; print "{$query1}<br>"; die("<p>Sorry ...</p>"); } $added_records = array(); $unresolved_pointers = array(); $missed_terms = array(); $missed_terms2 = array(); /*$detailTypes = getAllDetailTypeStructures(); //in current database $detailTypes = $detailTypes['typedefs']; $fld_ind = $detailTypes['fieldNamesToIndex']['dty_Type'] $detailTypes[$dttype][$fld_ind];*/ print "<br>************************************************<br>Import records"; print "<br>The following section adds records and allocates them new IDs."; if (!$rep_errors_only) { print "<br>It reports this in the form Old ID => New ID"; } // loop through the set of rectypes actually in the records in the database while ($row1 = mysql_fetch_array($res1)) { $rt = $row1[0]; if (!array_key_exists('cbr' . $rt, $_REQUEST)) { continue; } $recordType = $_REQUEST['cbr' . $rt]; if (intval($recordType) < 1) { print "<br>Record type {$rt} is not mapped"; ob_flush(); flush(); // flush to screen continue; } //@todo - add record type name $rt_counter = 0; print "<br>Record type: {$rt}"; // tell user somethign is happening ob_flush(); flush(); // flush to screen if ($is_h2) { $query2 = "select `rec_id`,`rec_url` from `{$sourcedb}`.`records` Where `{$sourcedb}`.`records`.`rec_type`={$rt}"; } else { $query2 = "select `rec_ID`,`rec_URL` from {$sourcedb}.Records Where {$sourcedb}.Records.rec_RecTypeID={$rt}"; } if ($user_rights) { $query2 = $query2 . " and " . $user_rights; } $res2 = mysql_query($query2); if (!$res2) { print "<div style='color:red;'>Bad query for records loop for source record type {$rt}</div>"; print "<br>Query: {$query2}"; ob_flush(); flush(); // flush to screen continue; //die ("<p>Sorry ...</p>"); } //special detailtype to keep original record id $dt_SourceRecordID = defined('DT_ORIGINAL_RECORD_ID') ? DT_ORIGINAL_RECORD_ID : 0; while ($row2 = mysql_fetch_array($res2)) { //select details and create details array $rid = $row2[0]; //record id // print "<br>".$rid." "; if ($is_h2) { $query3 = "SELECT `rd_type`, `rdt_type`, `rd_val`, `rd_file_id`, astext(`rd_geo`)\n\t\t\t\t\t\t\tFROM `{$sourcedb}`.`rec_details` rd, `{$sourcedb}`.`rec_detail_types` dt where rd.`rd_type`=dt.`rdt_id` and rd.`rd_rec_id`={$rid} order by `rd_type`"; } else { $query3 = "SELECT `dtl_DetailTypeID`, `dty_Type`, `dtl_Value`, `dtl_UploadedFileID`, astext(`dtl_Geo`)\n\t\t\t\t\t\t\tFROM {$sourcedb}.`recDetails` rd, {$sourcedb}.`defDetailTypes` dt where rd.`dtl_DetailTypeID`=dt.`dty_ID` and rd.`dtl_RecID`={$rid} order by `dtl_DetailTypeID`"; } $res3 = mysql_query($query3); // todo: check query was successful if (!$res3) { print "<br>record " . $rid . " <div style='color:red;'>bad select of detail fields</div>"; print "<br>query: {$query3}"; ob_flush(); flush(); // flush to screen continue; //die ("<p>Sorry ...</p>"); } $unresolved_records = array(); $details = array(); $dtid = 0; $key = 0; $ind = 0; $values = array(); //add special detail type 2-589 - reference to original record id if ($dt_SourceRecordID > 0) { $details["t:" . $dt_SourceRecordID] = array('0' => $rid); } while ($row3 = mysql_fetch_array($res3)) { if ($dtid != $row3[0]) { if ($key > 0) { $details["t:" . $key] = $values; } $dtid = $row3[0]; $values = array(); $ind; } if (!array_key_exists('cbd' . $row3[0], $_REQUEST)) { continue; } $key = $_REQUEST['cbd' . $row3[0]]; if (intval($key) < 1) { if ($rt == 52) { //debug print "mapping not defined for detail (field) #" . $dtid; } //mapping for this detail type is not specified continue; } $value = $row3[2]; //determine the type of filedtype if ($row3[1] == 'enum' || $row3[1] == 'relationtype') { if ($is_h2) { if (array_key_exists($value, $terms_h2)) { $value = $terms_h2[$value]; } else { if (array_search($value, $missed_terms) == false) { array_push($missed_terms, $value); } } } $term = getDestinationTerm($value); if ($term == null) { $ind = array_search(intval($value), $missed_terms2); if (count($missed_terms2) == 0 || $ind == 0 && $missed_terms2[$ind] != intval($value)) { array_push($missed_terms2, intval($value)); } $value = null; } else { $value = $term; } } else { if ($row3[1] == 'file') { if ($is_h2) { $value = copyRemoteFileH2($row3[3]); //returns new file id } else { $value = copyRemoteFile($row3[3]); //returns new file id } } else { if ($row3[1] == 'geo') { $value = $value . " " . $row3[4]; // string geotype+space+wkt } else { if ($row3[1] == 'relmarker') { } else { if ($row3[1] == 'resource') { //find the id of record in destionation database among pairs of added records if (array_key_exists($value, $added_records)) { $value = $added_records[$value]; } else { array_push($unresolved_records, $key . "|" . $value); //print "<div style='color:#ffaaaa;'>resource record#".$value." not found</div>"; $value = null; //ingnore } } } } } } if ($value != null) { $values[$ind] = $value; $ind++; } } //while by details //for last one if ($key > 0 && count($values) > 0) { $details["t:" . $key] = $values; } /*****DEBUG****/ //error_log("DETAILS:>>>>".print_r($details,true)); $ref = null; //add-update Heurist record $out = saveRecord(null, $recordType, $row2[1], null, null, null, null, null, null, null, null, $details, null, null, null, null, null, $ref, $ref, 2); if (@$out['error']) { print "<br>Source record# " . $rid . " "; print "=><div style='color:red'> Error: " . implode("; ", $out["error"]) . "</div>"; } else { $new_recid = $out["bibID"]; $added_records[$rid] = $new_recid; $rt_counter++; if (count($unresolved_records) > 0) { $unresolved_pointers[$new_recid] = $unresolved_records; } if (!$rep_errors_only) { print "<br>" . $rid . " => " . $out["bibID"]; if (@$out['warning']) { print "<br>Warning: " . implode("; ", $out["warning"]); } } } } //while by record of particular record type if ($rt_counter > 0) { print " => added {$rt_counter} records"; } ob_flush(); flush(); // flush to screen } // end of loop for record types /*****DEBUG****/ //error_log("DEBUG: UNRESOLVED POINTERS>>>>>".print_r($unresolved_pointers, true)); if (count($missed_terms) > 0) { print "<br><br>*********************************************************"; print "<br>These terms IDs are not found in {$sourcedb}<br>"; print implode('<br>', $missed_terms); } if (count($missed_terms2) > 0) { print "<br><br>*********************************************************"; print "<br>Mapping for these terms IDs is not specified<br>"; print implode('<br>', $missed_terms2); } if (count($unresolved_pointers) > 0) { $notfound_rec = array(); print "<br><br>*********************************************************"; print "<br>Finding and setting unresolved record pointers<br>"; if (!$rep_errors_only) { print "<br>It reports in form: source RecID => now target pointer RecID => in Rec Id<br>"; } //resolve record pointers $inserts = array(); foreach ($unresolved_pointers as $recid => $unresolved_records) { foreach ($unresolved_records as $code) { //print "<br>".$code; $aa = explode("|", $code); $dt_id = $aa[0]; $src_recid = $aa[1]; //print " ".$dt_id."=".$src_recid; if (array_key_exists($src_recid, $added_records)) { if (!$rep_errors_only) { print "<br>" . $src_recid . "=>" . $added_records[$src_recid] . "=>" . $recid; } array_push($inserts, "({$recid}, {$dt_id}, " . $added_records[$src_recid] . ", 1)"); } else { if (array_search($src_recid, $notfound_rec) == false) { array_push($notfound_rec, $src_recid . " for " . $recid); } } } } if (count($notfound_rec) > 0) { print "<br>These records are specified as pointers in source database but they were not added into target database:<br>"; print implode('<br>', $notfound_rec); } if (count($inserts) > 0) { //insert all new details $query1 = "insert into {$dbPrefix}" . HEURIST_DBNAME . ".recDetails (dtl_RecID, dtl_DetailTypeID, dtl_Value, dtl_AddedByImport) values " . join(",", $inserts); /*****DEBUG****/ //error_log(">>>>>>>>>>>>>>>".$query1); mysql_query($query1); print "<br><br>Total count of resolved pointers:" . count($inserts); } } print "<br><br><br><h3>Transfer completed - <a href=../../index.php?db=" . HEURIST_DBNAME . " title='Return to the main search page of the current database'><b>return to main page</b></a></h3>"; }
function doTransfer() { global $sourcedbname, $dbPrefix, $db_prefix, $is_h2, $user_id_insource, $user_workgroups; $sourcedb = $db_prefix . $sourcedbname; $rep_errors_only = @$_REQUEST['reportlevel'] == "1"; echo "<p>Now copying data from <b>{$sourcedb}</b> to <b>" . $dbPrefix . HEURIST_DBNAME . "</b><p>Processing: "; $terms_h2 = array(); $user_rights = null; // Loop through types for all records in the database (actual records, not defined types) if ($is_h2) { //load all terms $query1 = "SELECT `rdl_id`,`rdl_value` FROM `{$sourcedb}`.`rec_detail_lookups`"; $res1 = mysql_query($query1); while ($row1 = mysql_fetch_array($res1)) { $terms_h2[$row1[1]] = $row1[0]; } $query1 = "SELECT DISTINCT (`rec_type`) FROM `{$sourcedb}`.`records`"; $user_rights = "rec_temporary=0"; } else { $query1 = "SELECT DISTINCT (`rec_RecTypeID`) FROM {$sourcedb}.Records"; $user_rights = ' (rec_FlagTemporary=0 and (rec_OwnerUGrpID=' . $user_id_insource . ' or (not rec_NonOwnerVisibility="hidden")'; // rec_NonOwnerVisibility="public") if (!empty($user_workgroups)) { $user_rights = $user_rights . ' or rec_OwnerUGrpID in (' . join(',', $user_workgroups) . ')))'; } else { $user_rights = $user_rights . '))'; } } if ($user_rights) { $query1 = $query1 . " where " . $user_rights; } $res1 = mysql_query($query1); if (!$res1) { print "<br>Bad query for record type loop {$res1} <br>"; print "{$query1}<br>"; die("<p>Sorry ...</p>"); } $added_records = array(); $unresolved_pointers = array(); $missed_terms = array(); $missed_terms2 = array(); /*$detailTypes = getAllDetailTypeStructures(); //in current database $detailTypes = $detailTypes['typedefs']; $fld_ind = $detailTypes['fieldNamesToIndex']['dty_Type'] $detailTypes[$dttype][$fld_ind];*/ print "<br>************************************************<br>Import records"; print "<br>The following section adds records and allocates them new IDs."; if (!$rep_errors_only) { print "<br>It reports this in the form Old ID => New ID"; } // loop through the set of rectypes actually in the records in the database while ($row1 = mysql_fetch_array($res1)) { $rt = $row1[0]; if (!array_key_exists('cbr' . $rt, $_REQUEST)) { continue; } $recordType = $_REQUEST['cbr' . $rt]; if (intval($recordType) < 1) { print "<br><b>Record type {$rt} is not mapped</b>"; ob_flush(); flush(); // flush to screen continue; } //@todo - add record type name $rt_counter = 0; print "<br><br><br><b>Record type: {$rt}</b>"; // tell user something is happening ob_flush(); flush(); // flush to screen if ($is_h2) { // version 2 database $query2 = "select `rec_id`,`rec_url` from `{$sourcedb}`.`records` Where `{$sourcedb}`.`records`.`rec_type`={$rt}"; } else { // vedrsion 3 or later $query2 = "select `rec_ID`,`rec_URL` from {$sourcedb}.Records Where {$sourcedb}.Records.rec_RecTypeID={$rt}"; } if ($user_rights) { $query2 = $query2 . " and " . $user_rights; } $res2 = mysql_query($query2); if (!$res2) { print "<div style='color:red;'><b>Bad query for records loop for source record type {$rt}</b></div>"; print "<br>Query: {$query2}"; ob_flush(); flush(); // flush to screen continue; //die ("<p>Sorry ...</p>"); } //special detailtype to keep original record id $dt_SourceRecordID = defined('DT_ORIGINAL_RECORD_ID') ? DT_ORIGINAL_RECORD_ID : 0; while ($row2 = mysql_fetch_array($res2)) { //select details and create details array $rid = $row2[0]; //record id if ($is_h2) { // Heurist version 2 database $query3 = "SELECT `rd_type`, `rdt_type`, `rd_val`, `rd_file_id`, AsWKT(`rd_geo`)\n FROM `{$sourcedb}`.`rec_details` rd, `{$sourcedb}`.`rec_detail_types` dt where rd.`rd_type`=dt.`rdt_id` and rd.`rd_rec_id`={$rid} order by `rd_type`"; } else { // version 3 or later $query3 = "SELECT `dtl_DetailTypeID`, `dty_Type`, `dtl_Value`, `dtl_UploadedFileID`, AsWKT(`dtl_Geo`)\n FROM {$sourcedb}.`recDetails` rd, {$sourcedb}.`defDetailTypes` dt where rd.`dtl_DetailTypeID`=dt.`dty_ID` and rd.`dtl_RecID`={$rid} order by `dtl_DetailTypeID`"; } $res3 = mysql_query($query3); // todo: check query was successful if (!$res3) { print "<br>record " . $rid . " <div style='color:red;'><b>bad select query for detail fields, data for this record may not be correct</b></div>"; print "<br>query: {$query3}"; ob_flush(); flush(); // flush to screen continue; } $unresolved_records = array(); $details = array(); $dtid = 0; $key = 0; $ind = 0; $values = array(); //add special detail type 2-589 - reference to original record id if ($dt_SourceRecordID > 0) { $details["t:" . $dt_SourceRecordID] = array('0' => $rid); } while ($row3 = mysql_fetch_array($res3)) { if ($dtid != $row3[0]) { if ($key > 0) { $details["t:" . $key] = $values; } $dtid = $row3[0]; $values = array(); $ind; } if (!array_key_exists('cbd' . $row3[0], $_REQUEST)) { continue; } $key = $_REQUEST['cbd' . $row3[0]]; if (intval($key) < 1) { if ($rt == 52) { //debug TODO: What the hell is this? What is rt 52? print "mapping not defined for detail (field) #" . $dtid; } //mapping for this detail type is not specified continue; } $value = $row3[2]; //determine the type of field type if ($row3[1] == 'enum' || $row3[1] == 'relationtype') { if ($is_h2) { if (array_key_exists($value, $terms_h2)) { $value = $terms_h2[$value]; } else { if (array_search($value, $missed_terms) == false) { array_push($missed_terms, $value); } } } $term = getDestinationTerm($value); if ($term == null) { $ind = array_search(intval($value), $missed_terms2); if (count($missed_terms2) == 0 || $ind == 0 && $missed_terms2[$ind] != intval($value)) { array_push($missed_terms2, intval($value)); } $value = null; } else { $value = $term; } } else { if ($row3[1] == 'file') { if ($is_h2) { $value = copyRemoteFileH2($row3[3]); //returns new file id } else { $value = copyRemoteFile($row3[3]); //returns new file id } } else { if ($row3[1] == 'geo') { $value = $value . " " . $row3[4]; // string geotype+space+wkt } else { if ($row3[1] == 'relmarker') { // no data stored for relationship markers } else { if ($row3[1] == 'resource') { // record pointer field // find the id of record in destination database among pairs of added records if (array_key_exists($value, $added_records)) { $value = $added_records[$value]; } else { array_push($unresolved_records, $key . "|" . $value); //print "<div style='color:#ffaaaa;'>resource record#".$value." not found</div>"; $value = null; //ingnore } } } } } } if ($value != null) { $values[$ind] = $value; $ind++; } } //while by details //for last one if ($key > 0 && count($values) > 0) { $details["t:" . $key] = $values; } $ref = null; //add-update Heurist record $out = saveRecord(null, $recordType, $row2[1], null, null, null, null, null, null, null, null, $details, null, null, null, null, null, $ref, $ref, 2); if (@$out['error']) { print "<br>Source record# " . $rid . " "; print "=><div style='color:red'> Error: " . implode("; ", $out["error"]) . "</div>"; } else { $new_recid = $out["bibID"]; $added_records[$rid] = $new_recid; $rt_counter++; if (count($unresolved_records) > 0) { $unresolved_pointers[$new_recid] = $unresolved_records; } if (!$rep_errors_only) { // in active writing mode if (@$out['warning']) { print "<br>Warning: " . implode("; ", $out["warning"]); } print "<br>saved: rec#" . $rid . " => rec#" . $out["bibID"]; } } } //while by record of particular record type if ($rt_counter > 0) { print " => added {$rt_counter} records<br>"; } ob_flush(); flush(); // flush to screen } // end of loop for record types if (count($missed_terms) > 0) { print "<br><br>*********************************************************"; print "<br>These terms IDs are not found in {$sourcedb}<br>"; print implode('<br>', $missed_terms); } if (count($missed_terms2) > 0) { print "<br><br>*********************************************************"; print "<br>Mapping for these term IDs is not specified<br>"; print implode('<br>', $missed_terms2); } if (count($unresolved_pointers) > 0) { $notfound_rec = array(); print "<br><br>*********************************************************"; print "<br>Finding and setting unresolved record pointers<br>"; if (!$rep_errors_only) { print "<br>Report is in the form: source RecID => new target pointer RecID => in Rec Id<br>"; } //resolve record pointers $inserts = array(); foreach ($unresolved_pointers as $recid => $unresolved_records) { foreach ($unresolved_records as $code) { //print "<br>".$code; $aa = explode("|", $code); $dt_id = $aa[0]; $src_recid = $aa[1]; //print " ".$dt_id."=".$src_recid; if (array_key_exists($src_recid, $added_records)) { if (!$rep_errors_only) { print "<br>" . $src_recid . "=>" . $added_records[$src_recid] . "=>" . $recid; } array_push($inserts, "({$recid}, {$dt_id}, " . $added_records[$src_recid] . ", 1)"); } else { if (array_search($src_recid, $notfound_rec) == false) { array_push($notfound_rec, $src_recid . " for " . $recid); } } } } if (count($notfound_rec) > 0) { print "<br>These records are specified as pointers in source database but they were not added into target database:<br>"; print implode('<br>', $notfound_rec); } if (count($inserts) > 0) { //insert all new details $query1 = "insert into {$dbPrefix}" . HEURIST_DBNAME . ".recDetails (dtl_RecID, dtl_DetailTypeID, dtl_Value, dtl_AddedByImport) values " . join(",", $inserts); mysql_query($query1); print "<br><br>Total count of resolved pointers:" . count($inserts); } } //IMPORT TAGS/KEYWORDS ================================== // get all target users // find corresspondant source users // find all tags with records for source user // add tags to target tag table // add tag links in target for all imported records // get all target users $allusers = getUsers(); //in target database $rkeys = array_keys($_REQUEST); $tags_corr = array(); foreach ($rkeys as $rkey) { if (strpos($rkey, 'cbu') === 0 && @$allusers[$_REQUEST[$rkey]]) { // find corresspondant source users $source_uid = substr($rkey, 3); $target_uid = $_REQUEST[$rkey]; //find all tags with records for source user if ($is_h2) { $query1 = "select kwd.kwd_id, kwd.kwd_name, count(kwl_rec_id)\n from `{$sourcedb}`.`keywords` kwd, `{$sourcedb}`.`keyword_links` kwl\n where kwd.kwd_usr_id={$source_uid} and kwl.kwl_kwd_id=kwd.kwd_id\n group by kwd.kwd_id\n order by kwd.kwd_id"; } else { $query1 = "select kwd.tag_ID, kwd.tag_Text, count(rtl_RecID)\n from `{$sourcedb}`.`usrTags` kwd, `{$sourcedb}`.`usrRecTagLinks` kwl\n where kwd.tag_UGrpID={$source_uid} and kwl.rtl_TagID=kwd.tag_ID\n group by kwd.tag_ID\n order by kwd.tag_ID"; } //add tags to target tag table $res1 = mysql_query($query1); while ($row1 = mysql_fetch_array($res1)) { $ress = get_ids_for_tags(array($row1[1]), true, $target_uid); if (count($ress) > 0) { $tags_corr[$row1[0]] = array($ress[0], $row1[1], 0, $target_uid); } } } } //foreach users foreach ($added_records as $oldid => $newid) { // find tag links in source database if ($is_h2) { $query1 = "select kwl.kwl_kwd_id\n from `{$sourcedb}`.`keyword_links` kwl\n where kwl.kwl_rec_id=" . $oldid; } else { $query1 = "select kwl.rtl_TagID\n from `{$sourcedb}`.`usrRecTagLinks` kwl\n where kwl.rtl_RecID=" . $oldid; } $res1 = mysql_query($query1); while ($row1 = mysql_fetch_array($res1)) { $newtag = @$tags_corr[$row1[0]]; if ($newtag) { mysql_query('insert ignore into `' . USERS_DATABASE . '`.usrRecTagLinks (rtl_RecID, rtl_TagID, rtl_AddedByImport) ' . ' values (' . $newid . ',' . $newtag[0] . ',1)'); $tag_count = mysql_affected_rows(); if (mysql_error()) { } else { if ($tag_count > 0) { $tags_corr[$row1[0]][2]++; mysql_query('insert into `' . USERS_DATABASE . '`.usrBookmarks (bkm_UGrpID, bkm_Added, bkm_Modified, bkm_recID, bkm_AddedByImport) ' . 'values( ' . $newtag[3] . ', now(), now(), ' . $newid . ',1 )'); } } } } } //report print "<br />*********************************************************<br />"; print "Tags added<br /><br />"; foreach ($tags_corr as $oldid => $newtag) { print $oldid . " " . $newtag[1] . " ==> " . $newtag[0] . ", " . $newtag[2] . " records<br />"; } print "<br /><br />"; //FINAL ================================== print "<br><br><br><h3>Transfer completed - <a href=getRecordsFromDB.php?db=" . HEURIST_DBNAME . " title='Return to the main search page of the current database'><b>return to main page</b></a></h3>"; }