/** * @param $userid * @param $headers * @param $table_csv * @param array $fields * @param $parent_chkd_flds * @param $export_file_name * @param $debug * @param null $comment * @param array $to */ public static function do_sendit($userid, $headers, $table_csv, $fields = array(), $parent_chkd_flds, $export_file_name, $comment = null, $to = array(), $debug) { global $project_id, $user_rights, $app_title, $lang, $redcap_version; // we could use the global $userid, but we need control of it for setting the user as [CRON], so this is passed in args. $return_val = false; $export_type = 0; // this puts all files generated here in the Data Export category in the File Repository $today = date("Y-m-d_Hi"); //get today for filename $projTitleShort = substr(str_replace(" ", "", ucwords(preg_replace("/[^a-zA-Z0-9 ]/", "", html_entity_decode($app_title, ENT_QUOTES)))), 0, 20); // shortened project title for filename $originalFilename = $projTitleShort . "_" . $export_file_name . "_DATA_" . $today . ".csv"; // name the file for storage $today = date("Y-m-d-H-i-s"); // get today for comment, subsequent processing as needed $docs_comment_WH = $export_type ? "Data export file created by $userid on $today" : fix_case($export_file_name) . " file created by $userid on $today. $comment"; // unused, but I keep it around just in case /** * setup vars for value export logging */ $chkd_fields = implode(',', $fields); /** * turn on/off exporting per user rights */ if (($user_rights['data_export_tool'] || $userid == '[CRON]') && !$debug) { $table_csv = addBOMtoUTF8($headers . $table_csv); /** * Store the file in the file system and log the activity, handle if error */ if (!DataExport::storeExportFile($originalFilename, $table_csv, true)) { log_event("", "redcap_data", "data_export", "", str_replace("'", "", $chkd_fields) . (($parent_chkd_flds == "") ? "" : ", " . str_replace("'", "", $parent_chkd_flds)), "Data Export Failed"); } else { log_event("", "redcap_data", "data_export", "", str_replace("'", "", $chkd_fields) . (($parent_chkd_flds == "") ? "" : ", " . str_replace("'", "", $parent_chkd_flds)), "Export data for SendIt"); /** * email file link and download password in two separate emails via REDCap SendIt */ $file_info_sql = db_query("SELECT docs_id, docs_size, docs_type FROM redcap_docs WHERE project_id = $project_id ORDER BY docs_id DESC LIMIT 1"); // get required info about the file we just created if ($file_info_sql) { $docs_id = db_result($file_info_sql, 0, 'docs_id'); $docs_size = db_result($file_info_sql, 0, 'docs_size'); $docs_type = db_result($file_info_sql, 0, 'docs_type'); } $yourName = 'PRIORITIZE REDCap'; $expireDays = 3; // set the SendIt to expire in this many days /** * $file_location: * 1 = ephemeral, will be deleted on $expireDate * 2 = export file, visible only to rights in file repository */ $file_location = 2; $send = 1; // always send download confirmation $expireDate = date('Y-m-d H:i:s', strtotime("+$expireDays days")); $expireYear = substr($expireDate, 0, 4); $expireMonth = substr($expireDate, 5, 2); $expireDay = substr($expireDate, 8, 2); $expireHour = substr($expireDate, 11, 2); $expireMin = substr($expireDate, 14, 2); // Add entry to sendit_docs table $query = "INSERT INTO redcap_sendit_docs (doc_name, doc_orig_name, doc_type, doc_size, send_confirmation, expire_date, username, location, docs_id, date_added) VALUES ('$originalFilename', '" . prep($originalFilename) . "', '$docs_type', '$docs_size', $send, '$expireDate', '" . prep($userid) . "', $file_location, $docs_id, '" . NOW . "')"; db_query($query); $newId = db_insert_id(); $logDescrip = "Send file from file repository (Send-It)"; log_event($query, "redcap_sendit_docs", "MANAGE", $newId, "document_id = $newId", $logDescrip); // Set email subject $subject = "[PRIORITIZE] " . $comment; $subject = html_entity_decode($subject, ENT_QUOTES); // Set email From address $from = array('Ken Bergquist' => '*****@*****.**'); // Begin set up of email to send to recipients $email = new Message(); foreach ($from as $name => $address) { $email->setFrom($address); $email->setFromName($name); } $email->setSubject($subject); // Loop through each recipient and send email foreach ($to as $name => $address) { // If a non-blank email address if (trim($address) != '') { // create key for unique url $key = strtoupper(substr(uniqid(md5(mt_rand())), 0, 25)); // create password $pwd = generateRandomHash(8, false, true); $query = "INSERT INTO redcap_sendit_recipients (email_address, sent_confirmation, download_date, download_count, document_id, guid, pwd) VALUES ('$address', 0, NULL, 0, $newId, '$key', '" . md5($pwd) . "')"; $q = db_query($query); // Download URL $url = APP_PATH_WEBROOT_FULL . 'redcap_v' . $redcap_version . '/SendIt/download.php?' . $key; // Message from sender $note = "$comment for $today"; // Get YMD timestamp of the file's expiration time $expireTimestamp = date('Y-m-d H:i:s', mktime($expireHour, $expireMin, 0, $expireMonth, $expireDay, $expireYear)); // Email body $body = "<html><body style=\"font-family:Arial;font-size:10pt;\"> $yourName {$lang['sendit_51']} \"$originalFilename\" {$lang['sendit_52']} " . date('l', mktime($expireHour, $expireMin, 0, $expireMonth, $expireDay, $expireYear)) . ", " . DateTimeRC::format_ts_from_ymd($expireTimestamp) . "{$lang['period']} {$lang['sendit_53']}<br><br> {$lang['sendit_54']}<br> <a href=\"$url\">$url</a><br><br> $note <br>-----------------------------------------------<br> {$lang['sendit_55']} " . CONSORTIUM_WEBSITE_DOMAIN . ". </body></html>"; // Construct email and send $email->setTo($address); $email->setToName($name); $email->setBody($body); if ($email->send()) { // Now send follow-up email containing password $bodypass = "******"font-family:Arial;font-size:10pt;\"> {$lang['sendit_50']}<br><br> $pwd<br><br> </body></html>"; $email->setSubject("Re: $subject"); $email->setBody($bodypass); sleep(2); // Hold for a second so that second email somehow doesn't reach the user first $email->send(); } else { error_log("ERROR: pid=$project_id: Email to $name <$address> NOT SENT"); } } } } unset($table_csv); } }
public static function storeExportFile($original_filename, $file_content, $archiveFile = false, $dateShiftDates = false) { global $edoc_storage_option; ## Create the stored name of the file as it wll be stored in the file system $stored_name = date('YmdHis') . "_pid" . PROJECT_ID . "_" . generateRandomHash(6) . getFileExt($original_filename, true); $file_extension = getFileExt($original_filename); $mime_type = strtolower($file_extension) == 'csv' ? 'application/csv' : 'application/octet-stream'; // If file is UTF-8 encoded, then add BOM // Do NOT use addBOMtoUTF8() on Stata syntax file (.do) because BOM causes issues in syntax file if (strtolower($file_extension) != 'do') { $file_content = addBOMtoUTF8($file_content); } // If Gzip enabled, then gzip the file and append filename with .gz extension list($file_content, $stored_name, $gzipped) = gzip_encode_file($file_content, $stored_name); // Get file size in bytes $docs_size = strlen($file_content); // Add file to file system if ($edoc_storage_option == '0') { // Store locally $fp = fopen(EDOC_PATH . $stored_name, 'w'); if ($fp !== false && fwrite($fp, $file_content) !== false) { // Close connection fclose($fp); } else { // Send error response return false; } // Add file to S3 } elseif ($edoc_storage_option == '2') { global $amazon_s3_key, $amazon_s3_secret, $amazon_s3_bucket; $s3 = new S3($amazon_s3_key, $amazon_s3_secret, SSL); if (!$s3->putObject($file_content, $amazon_s3_bucket, $stored_name, S3::ACL_PUBLIC_READ_WRITE)) { // Send error response return false; } } else { // Store using WebDAV require_once APP_PATH_CLASSES . "WebdavClient.php"; require APP_PATH_WEBTOOLS . 'webdav/webdav_connection.php'; $wdc = new WebdavClient(); $wdc->set_server($webdav_hostname); $wdc->set_port($webdav_port); $wdc->set_ssl($webdav_ssl); $wdc->set_user($webdav_username); $wdc->set_pass($webdav_password); $wdc->set_protocol(1); // use HTTP/1.1 $wdc->set_debug(false); // enable debugging? if (!$wdc->open()) { // Send error response return false; } if (substr($webdav_path, -1) != '/') { $webdav_path .= '/'; } $http_status = $wdc->put($webdav_path . $stored_name, $file_content); $wdc->close(); } ## Add file info to edocs_metadata table // If not archiving file in File Repository, then set to be deleted in 1 hour $delete_time = $archiveFile ? "" : NOW; // Add to table $sql = "insert into redcap_edocs_metadata (stored_name, mime_type, doc_name, doc_size, file_extension, project_id, \n\t\t\t\tstored_date, delete_date, gzipped) values ('" . prep($stored_name) . "', '{$mime_type}', '" . prep($original_filename) . "', \n\t\t\t\t'" . prep($docs_size) . "', '" . prep($file_extension) . "', " . PROJECT_ID . ", '" . NOW . "', " . checkNull($delete_time) . ", {$gzipped})"; if (!db_query($sql)) { // Send error response return false; } // Get edoc_id $edoc_id = db_insert_id(); ## Add to doc_to_edoc table // Set flag if data is date shifted $dateShiftFlag = $dateShiftDates ? "DATE_SHIFT" : ""; // Set "comment" in docs table if (strtolower($file_extension) == 'csv') { $docs_comment = "Data export file created by " . USERID . " on " . date("Y-m-d-H-i-s"); } else { if ($file_extension == 'sps') { $stats_package_name = 'Spss'; } elseif ($file_extension == 'do') { $stats_package_name = 'Stata'; } else { $stats_package_name = camelCase($file_extension); } $docs_comment = "{$stats_package_name} syntax file created by " . USERID . " on " . date("Y-m-d-H-i-s"); } // Archive in redcap_docs table $sql = "INSERT INTO redcap_docs (project_id, docs_name, docs_file, docs_date, docs_size, docs_comment, docs_type, \n\t\t\t\tdocs_rights, export_file, temp) VALUES (" . PROJECT_ID . ", '" . prep($original_filename) . "', NULL, '" . TODAY . "', \n\t\t\t\t'{$docs_size}', '" . prep($docs_comment) . "', '{$mime_type}', " . checkNull($dateShiftFlag) . ", 1, \n\t\t\t\t" . checkNull($archiveFile ? "0" : "1") . ")"; if (db_query($sql)) { $docs_id = db_insert_id(); // Add to redcap_docs_to_edocs also $sql = "insert into redcap_docs_to_edocs (docs_id, doc_id) values ({$docs_id}, {$edoc_id})"; db_query($sql); } else { // Could not store in table, so remove from edocs_metadata also db_query("delete from redcap_edocs_metadata where doc_id = {$edoc_id}"); return false; } // Return successful response of docs_id from redcap_docs table return $docs_id; }
$ids = array_keys($ids); $id_count = count($ids); //echo "Records are: <pre>" . print_r($ids,true) . "</pre>"; //echo "\nRecord count is " . count($ids); d($ids); // Determine batching size (with a minimum of 1 record per export). $records_per_batch = max(1,floor($fields_per_batch / $field_count)); $batches = array_chunk($ids,$records_per_batch); $batch_total = count($batches); //print "<pre>" . print_r($batches,true) . "</pre>"; d($batches); // Create temp file - Set the target file to be saved in the temp dir (set timestamp in filename as 1 hour from now so that it gets deleted automatically in 1 hour) $inOneHour = date("YmdHis", mktime(date("H")+1,date("i"),date("s"),date("m"),date("d"),date("Y"))); $target_filename = "{$inOneHour}_pid{$project_id}_".generateRandomHash(6).".csv"; $target_file = APP_PATH_TEMP . $target_filename; echo RCView::div(array('class'=>'round chklist','id'=>'Large Data Export'), RCView::div(array('class'=>'chklisthdr','style'=>'color:rgb(128,0,0);margin-top:10px;'), "Exporting Complete CSV"). RCView::p(array(),"Breaking $id_count records into $batch_total batch exports..."). RCView::div(array('id'=>'progress','style'=>'width:600px;border:1px solid #ccc;margin-bottom:10px;')). RCView::div(array('id'=>'progress_info','style'=>'width')) ); if (!$debug) { // Start Export REDCap::logEvent("Full Export Requested"); $fh = fopen($target_file, 'w') or die("can't open file"); $time_start = microtime(true); $batch_times = array();
unlink($dummy['tmp_name']); // Set error msg $errors[] = $lang['sendit_03'] . ' (' . round_up($dummy['size']/1024/1024) . ' MB) ' . $lang['sendit_04'] . ' ' . maxUploadSizeFileRespository() . ' MB ' . $lang['sendit_05']; } if (strlen($dummy['tmp_name']) > 0 && empty($errors)) { $dummy_tmp_name = $dummy['tmp_name']; $dummy_file_size = $dummy['size']; $dummy_file_type = $dummy['type']; $dummy_file_name = $dummy['name']; $dummy_file_name = preg_replace("/[^a-zA-Z-._0-9]/","_",$dummy_file_name); $dummy_file_name = str_replace("__","_",$dummy_file_name); $dummy_file_name = str_replace("__","_",$dummy_file_name); $file_extension = getFileExt($dummy_file_name); $stored_name = date('YmdHis') . "_pid" . $project_id . "_" . generateRandomHash(6) . getFileExt($dummy_file_name, true); if ($edoc_storage_option == '1') { // Webdav $dummy_file_content = file_get_contents($dummy['tmp_name']); $upload_success = ($wdc->put($webdav_path . $stored_name, $dummy_file_content) == '201'); } elseif ($edoc_storage_option == '2') { // S3 $s3 = new S3($amazon_s3_key, $amazon_s3_secret, SSL); $upload_success = ($s3->putObjectFile($dummy['tmp_name'], $amazon_s3_bucket, $stored_name, S3::ACL_PUBLIC_READ_WRITE)); } else {