/**
* @param $userid
* @param $headers
* @param $table_csv
* @param array $fields
* @param $parent_chkd_flds
* @param $export_file_name
* @param $debug
* @param null $comment
* @param array $to
*/
public static function do_sendit($userid, $headers, $table_csv, $fields = array(), $parent_chkd_flds, $export_file_name, $comment = null, $to = array(), $debug)
{
global $project_id, $user_rights, $app_title, $lang, $redcap_version; // we could use the global $userid, but we need control of it for setting the user as [CRON], so this is passed in args.
$return_val = false;
$export_type = 0; // this puts all files generated here in the Data Export category in the File Repository
$today = date("Y-m-d_Hi"); //get today for filename
$projTitleShort = substr(str_replace(" ", "", ucwords(preg_replace("/[^a-zA-Z0-9 ]/", "", html_entity_decode($app_title, ENT_QUOTES)))), 0, 20); // shortened project title for filename
$originalFilename = $projTitleShort . "_" . $export_file_name . "_DATA_" . $today . ".csv"; // name the file for storage
$today = date("Y-m-d-H-i-s"); // get today for comment, subsequent processing as needed
$docs_comment_WH = $export_type ? "Data export file created by $userid on $today" : fix_case($export_file_name) . " file created by $userid on $today. $comment"; // unused, but I keep it around just in case
/**
* setup vars for value export logging
*/
$chkd_fields = implode(',', $fields);
/**
* turn on/off exporting per user rights
*/
if (($user_rights['data_export_tool'] || $userid == '[CRON]') && !$debug) {
$table_csv = addBOMtoUTF8($headers . $table_csv);
/**
* Store the file in the file system and log the activity, handle if error
*/
if (!DataExport::storeExportFile($originalFilename, $table_csv, true)) {
log_event("", "redcap_data", "data_export", "", str_replace("'", "", $chkd_fields) . (($parent_chkd_flds == "") ? "" : ", " . str_replace("'", "", $parent_chkd_flds)), "Data Export Failed");
} else {
log_event("", "redcap_data", "data_export", "", str_replace("'", "", $chkd_fields) . (($parent_chkd_flds == "") ? "" : ", " . str_replace("'", "", $parent_chkd_flds)), "Export data for SendIt");
/**
* email file link and download password in two separate emails via REDCap SendIt
*/
$file_info_sql = db_query("SELECT docs_id, docs_size, docs_type FROM redcap_docs WHERE project_id = $project_id ORDER BY docs_id DESC LIMIT 1"); // get required info about the file we just created
if ($file_info_sql) {
$docs_id = db_result($file_info_sql, 0, 'docs_id');
$docs_size = db_result($file_info_sql, 0, 'docs_size');
$docs_type = db_result($file_info_sql, 0, 'docs_type');
}
$yourName = 'PRIORITIZE REDCap';
$expireDays = 3; // set the SendIt to expire in this many days
/**
* $file_location:
* 1 = ephemeral, will be deleted on $expireDate
* 2 = export file, visible only to rights in file repository
*/
$file_location = 2;
$send = 1; // always send download confirmation
$expireDate = date('Y-m-d H:i:s', strtotime("+$expireDays days"));
$expireYear = substr($expireDate, 0, 4);
$expireMonth = substr($expireDate, 5, 2);
$expireDay = substr($expireDate, 8, 2);
$expireHour = substr($expireDate, 11, 2);
$expireMin = substr($expireDate, 14, 2);
// Add entry to sendit_docs table
$query = "INSERT INTO redcap_sendit_docs (doc_name, doc_orig_name, doc_type, doc_size, send_confirmation, expire_date, username,
location, docs_id, date_added)
VALUES ('$originalFilename', '" . prep($originalFilename) . "', '$docs_type', '$docs_size', $send, '$expireDate', '" . prep($userid) . "',
$file_location, $docs_id, '" . NOW . "')";
db_query($query);
$newId = db_insert_id();
$logDescrip = "Send file from file repository (Send-It)";
log_event($query, "redcap_sendit_docs", "MANAGE", $newId, "document_id = $newId", $logDescrip);
// Set email subject
$subject = "[PRIORITIZE] " . $comment;
$subject = html_entity_decode($subject, ENT_QUOTES);
// Set email From address
$from = array('Ken Bergquist' => '*****@*****.**');
// Begin set up of email to send to recipients
$email = new Message();
foreach ($from as $name => $address) {
$email->setFrom($address);
$email->setFromName($name);
}
$email->setSubject($subject);
// Loop through each recipient and send email
foreach ($to as $name => $address) {
// If a non-blank email address
if (trim($address) != '') {
// create key for unique url
$key = strtoupper(substr(uniqid(md5(mt_rand())), 0, 25));
// create password
$pwd = generateRandomHash(8, false, true);
$query = "INSERT INTO redcap_sendit_recipients (email_address, sent_confirmation, download_date, download_count, document_id, guid, pwd)
VALUES ('$address', 0, NULL, 0, $newId, '$key', '" . md5($pwd) . "')";
$q = db_query($query);
// Download URL
$url = APP_PATH_WEBROOT_FULL . 'redcap_v' . $redcap_version . '/SendIt/download.php?' . $key;
// Message from sender
$note = "$comment for $today";
// Get YMD timestamp of the file's expiration time
$expireTimestamp = date('Y-m-d H:i:s', mktime($expireHour, $expireMin, 0, $expireMonth, $expireDay, $expireYear));
// Email body
$body = "<html><body style=\"font-family:Arial;font-size:10pt;\">
$yourName {$lang['sendit_51']} \"$originalFilename\" {$lang['sendit_52']} " .
date('l', mktime($expireHour, $expireMin, 0, $expireMonth, $expireDay, $expireYear)) . ",
" . DateTimeRC::format_ts_from_ymd($expireTimestamp) . "{$lang['period']}
{$lang['sendit_53']}<br><br>
{$lang['sendit_54']}<br>
<a href=\"$url\">$url</a><br><br>
$note
<br>-----------------------------------------------<br>
{$lang['sendit_55']} " . CONSORTIUM_WEBSITE_DOMAIN . ".
</body></html>";
// Construct email and send
$email->setTo($address);
$email->setToName($name);
$email->setBody($body);
if ($email->send()) {
// Now send follow-up email containing password
$bodypass = "******"font-family:Arial;font-size:10pt;\">
{$lang['sendit_50']}<br><br>
$pwd<br><br>
</body></html>";
$email->setSubject("Re: $subject");
$email->setBody($bodypass);
sleep(2); // Hold for a second so that second email somehow doesn't reach the user first
$email->send();
} else {
error_log("ERROR: pid=$project_id: Email to $name <$address> NOT SENT");
}
}
}
}
unset($table_csv);
}
}
public static function storeExportFile($original_filename, $file_content, $archiveFile = false, $dateShiftDates = false)
{
global $edoc_storage_option;
## Create the stored name of the file as it wll be stored in the file system
$stored_name = date('YmdHis') . "_pid" . PROJECT_ID . "_" . generateRandomHash(6) . getFileExt($original_filename, true);
$file_extension = getFileExt($original_filename);
$mime_type = strtolower($file_extension) == 'csv' ? 'application/csv' : 'application/octet-stream';
// If file is UTF-8 encoded, then add BOM
// Do NOT use addBOMtoUTF8() on Stata syntax file (.do) because BOM causes issues in syntax file
if (strtolower($file_extension) != 'do') {
$file_content = addBOMtoUTF8($file_content);
}
// If Gzip enabled, then gzip the file and append filename with .gz extension
list($file_content, $stored_name, $gzipped) = gzip_encode_file($file_content, $stored_name);
// Get file size in bytes
$docs_size = strlen($file_content);
// Add file to file system
if ($edoc_storage_option == '0') {
// Store locally
$fp = fopen(EDOC_PATH . $stored_name, 'w');
if ($fp !== false && fwrite($fp, $file_content) !== false) {
// Close connection
fclose($fp);
} else {
// Send error response
return false;
}
// Add file to S3
} elseif ($edoc_storage_option == '2') {
global $amazon_s3_key, $amazon_s3_secret, $amazon_s3_bucket;
$s3 = new S3($amazon_s3_key, $amazon_s3_secret, SSL);
if (!$s3->putObject($file_content, $amazon_s3_bucket, $stored_name, S3::ACL_PUBLIC_READ_WRITE)) {
// Send error response
return false;
}
} else {
// Store using WebDAV
require_once APP_PATH_CLASSES . "WebdavClient.php";
require APP_PATH_WEBTOOLS . 'webdav/webdav_connection.php';
$wdc = new WebdavClient();
$wdc->set_server($webdav_hostname);
$wdc->set_port($webdav_port);
$wdc->set_ssl($webdav_ssl);
$wdc->set_user($webdav_username);
$wdc->set_pass($webdav_password);
$wdc->set_protocol(1);
// use HTTP/1.1
$wdc->set_debug(false);
// enable debugging?
if (!$wdc->open()) {
// Send error response
return false;
}
if (substr($webdav_path, -1) != '/') {
$webdav_path .= '/';
}
$http_status = $wdc->put($webdav_path . $stored_name, $file_content);
$wdc->close();
}
## Add file info to edocs_metadata table
// If not archiving file in File Repository, then set to be deleted in 1 hour
$delete_time = $archiveFile ? "" : NOW;
// Add to table
$sql = "insert into redcap_edocs_metadata (stored_name, mime_type, doc_name, doc_size, file_extension, project_id, \n\t\t\t\tstored_date, delete_date, gzipped) values ('" . prep($stored_name) . "', '{$mime_type}', '" . prep($original_filename) . "', \n\t\t\t\t'" . prep($docs_size) . "', '" . prep($file_extension) . "', " . PROJECT_ID . ", '" . NOW . "', " . checkNull($delete_time) . ", {$gzipped})";
if (!db_query($sql)) {
// Send error response
return false;
}
// Get edoc_id
$edoc_id = db_insert_id();
## Add to doc_to_edoc table
// Set flag if data is date shifted
$dateShiftFlag = $dateShiftDates ? "DATE_SHIFT" : "";
// Set "comment" in docs table
if (strtolower($file_extension) == 'csv') {
$docs_comment = "Data export file created by " . USERID . " on " . date("Y-m-d-H-i-s");
} else {
if ($file_extension == 'sps') {
$stats_package_name = 'Spss';
} elseif ($file_extension == 'do') {
$stats_package_name = 'Stata';
} else {
$stats_package_name = camelCase($file_extension);
}
$docs_comment = "{$stats_package_name} syntax file created by " . USERID . " on " . date("Y-m-d-H-i-s");
}
// Archive in redcap_docs table
$sql = "INSERT INTO redcap_docs (project_id, docs_name, docs_file, docs_date, docs_size, docs_comment, docs_type, \n\t\t\t\tdocs_rights, export_file, temp) VALUES (" . PROJECT_ID . ", '" . prep($original_filename) . "', NULL, '" . TODAY . "', \n\t\t\t\t'{$docs_size}', '" . prep($docs_comment) . "', '{$mime_type}', " . checkNull($dateShiftFlag) . ", 1, \n\t\t\t\t" . checkNull($archiveFile ? "0" : "1") . ")";
if (db_query($sql)) {
$docs_id = db_insert_id();
// Add to redcap_docs_to_edocs also
$sql = "insert into redcap_docs_to_edocs (docs_id, doc_id) values ({$docs_id}, {$edoc_id})";
db_query($sql);
} else {
// Could not store in table, so remove from edocs_metadata also
db_query("delete from redcap_edocs_metadata where doc_id = {$edoc_id}");
return false;
}
// Return successful response of docs_id from redcap_docs table
return $docs_id;
}
$ids = array_keys($ids);
$id_count = count($ids);
//echo "Records are: <pre>" . print_r($ids,true) . "</pre>";
//echo "\nRecord count is " . count($ids);
d($ids);
// Determine batching size (with a minimum of 1 record per export).
$records_per_batch = max(1,floor($fields_per_batch / $field_count));
$batches = array_chunk($ids,$records_per_batch);
$batch_total = count($batches);
//print "<pre>" . print_r($batches,true) . "</pre>";
d($batches);
// Create temp file - Set the target file to be saved in the temp dir (set timestamp in filename as 1 hour from now so that it gets deleted automatically in 1 hour)
$inOneHour = date("YmdHis", mktime(date("H")+1,date("i"),date("s"),date("m"),date("d"),date("Y")));
$target_filename = "{$inOneHour}_pid{$project_id}_".generateRandomHash(6).".csv";
$target_file = APP_PATH_TEMP . $target_filename;
echo RCView::div(array('class'=>'round chklist','id'=>'Large Data Export'),
RCView::div(array('class'=>'chklisthdr','style'=>'color:rgb(128,0,0);margin-top:10px;'), "Exporting Complete CSV").
RCView::p(array(),"Breaking $id_count records into $batch_total batch exports...").
RCView::div(array('id'=>'progress','style'=>'width:600px;border:1px solid #ccc;margin-bottom:10px;')).
RCView::div(array('id'=>'progress_info','style'=>'width'))
);
if (!$debug) { // Start Export
REDCap::logEvent("Full Export Requested");
$fh = fopen($target_file, 'w') or die("can't open file");
$time_start = microtime(true);
$batch_times = array();
unlink($dummy['tmp_name']);
// Set error msg
$errors[] = $lang['sendit_03'] . ' (' . round_up($dummy['size']/1024/1024) . ' MB) ' .
$lang['sendit_04'] . ' ' . maxUploadSizeFileRespository() . ' MB ' . $lang['sendit_05'];
}
if (strlen($dummy['tmp_name']) > 0 && empty($errors))
{
$dummy_tmp_name = $dummy['tmp_name'];
$dummy_file_size = $dummy['size'];
$dummy_file_type = $dummy['type'];
$dummy_file_name = $dummy['name'];
$dummy_file_name = preg_replace("/[^a-zA-Z-._0-9]/","_",$dummy_file_name);
$dummy_file_name = str_replace("__","_",$dummy_file_name);
$dummy_file_name = str_replace("__","_",$dummy_file_name);
$file_extension = getFileExt($dummy_file_name);
$stored_name = date('YmdHis') . "_pid" . $project_id . "_" . generateRandomHash(6) . getFileExt($dummy_file_name, true);
if ($edoc_storage_option == '1')
{
// Webdav
$dummy_file_content = file_get_contents($dummy['tmp_name']);
$upload_success = ($wdc->put($webdav_path . $stored_name, $dummy_file_content) == '201');
}
elseif ($edoc_storage_option == '2')
{
// S3
$s3 = new S3($amazon_s3_key, $amazon_s3_secret, SSL);
$upload_success = ($s3->putObjectFile($dummy['tmp_name'], $amazon_s3_bucket, $stored_name, S3::ACL_PUBLIC_READ_WRITE));
}
else
{
