/**
* Test for an export path
*/
public function test_version1invalidexportpath()
{
global $CFG, $DB, $USER;
// Check if test is being run as root.
if (posix_getuid() === 0) {
$this->markTestSkipped('This test will always fail when run as root.');
}
require_once $CFG->dirroot . '/local/datahub/fileplugins/log/log.class.php';
require_once $CFG->dirroot . '/local/datahub/lib.php';
set_config('export_path', 'invalidexportpath', 'dhexport_version1');
$filepath = $CFG->dataroot . '/invalidexportpath';
// Create a folder and make it executable only.
// Cleanup the folder first if it already exists.
if (file_exists($filepath)) {
// Remove any files.
if (!empty($filepath)) {
foreach (glob("{$filepath}/*") as $logfile) {
unlink($logfile);
}
}
rmdir($filepath);
}
mkdir($filepath, 0100);
// Set up the export file path.
$filename = 'rliptestexport.csv';
set_config('export_file', $filename, 'dhexport_version1');
// Set up data for one course and one enroled user.
$this->load_csv_data();
// Create a scheduled job.
$data = array('plugin' => 'dhexport_version1', 'period' => '5m', 'label' => 'bogus', 'type' => 'dhexport');
$taskid = rlip_schedule_add_job($data);
// Change the next runtime to a known value in the past.
$task = new stdClass();
$task->id = $taskid;
$task->nextruntime = 99;
$DB->update_record('local_eliscore_sched_tasks', $task);
$job = new stdClass();
$job->id = $DB->get_field(RLIP_SCHEDULE_TABLE, 'id', array('plugin' => 'dhexport_version1'));
$job->nextruntime = 99;
$DB->update_record(RLIP_SCHEDULE_TABLE, $job);
// Lower bound on starttime.
$starttime = time();
// Run the export.
$taskname = $DB->get_field('local_eliscore_sched_tasks', 'taskname', array('id' => $taskid));
run_ipjob($taskname);
// Database error log validation.
$dataroot = rtrim($CFG->dataroot, DIRECTORY_SEPARATOR);
$select = "{$DB->sql_compare_text('statusmessage')} = :message";
$msgparam = "Export file rliptestexport.csv cannot be processed because the folder: {$dataroot}/invalidexportpath/ is not";
$msgparam .= " accessible. Please fix the export path.";
$params = array('message' => $msgparam);
$exists = $DB->record_exists_select(RLIP_LOG_TABLE, $select, $params);
// Cleanup the folder.
if (file_exists($filepath)) {
// Remove any files.
if (!empty($filepath)) {
foreach (glob("{$filepath}/*") as $logfile) {
unlink($logfile);
}
}
rmdir($filepath);
}
$this->assertEquals($exists, true);
// Fs logger error log validation.
$dataroot = rtrim($CFG->dataroot, DIRECTORY_SEPARATOR);
$expectederror = "Export file rliptestexport.csv cannot be processed because the folder: {$dataroot}/invalidexportpath/ is";
$expectederror .= " not accessible. Please fix the export path.\n";
// Validate that a log file was created.
$plugintype = 'export';
$plugin = 'dhexport_version1';
$format = get_string('logfile_timestamp', 'local_datahub');
// Get most recent record.
$records = $DB->get_records(RLIP_LOG_TABLE, null, 'starttime DESC');
foreach ($records as $record) {
$logfile = $record->logpath;
break;
}
$testfilename = $logfile;
$filename = self::get_current_logfile($testfilename);
$this->assertTrue(file_exists($filename));
// Fetch log line.
$pointer = fopen($filename, 'r');
$line = fgets($pointer);
fclose($pointer);
if ($line == false) {
// No line found.
$this->assertEquals(0, 1);
}
// Data validation.
$prefixlength = strlen('[MMM/DD/YYYY:hh:mm:ss -zzzz] ');
$actualerror = substr($line, $prefixlength);
$this->assertEquals($expectederror, $actualerror);
}
/**
* Validate that scheduled import is prevented if existing incomplete run exists.
*/
public function test_importpreventmultipleimports()
{
global $CFG, $DB;
require_once $CFG->dirroot . '/local/datahub/lib.php';
require_once $CFG->dirroot . '/local/eliscore/lib/tasklib.php';
$DB->delete_records('local_eliscore_sched_tasks');
$DB->delete_records(RLIP_SCHEDULE_TABLE);
$filepath = '/local_datahub_phpunit/';
$filename = 'userfile2.csv';
set_config('schedule_files_path', $filepath, 'dhimport_version1elis');
set_config('user_schedule_file', $filename, 'dhimport_version1elis');
// Set up the test directory.
$testdir = $CFG->dataroot . $filepath;
mkdir($testdir, 0777, true);
copy(dirname(__FILE__) . "/fixtures/{$filename}", $testdir . $filename);
// Create the job.
$data = array('plugin' => 'dhimport_version1elis', 'period' => '5m', 'type' => 'dhimport');
$taskid1 = rlip_schedule_add_job($data);
// Run the import with a time in the past so it stops immediately.
$taskname1 = $DB->get_field('local_eliscore_sched_tasks', 'taskname', array('id' => $taskid1));
$result1 = run_ipjob($taskname1, -1);
// Validate the first import run was started.
$this->assertEquals(true, $result1);
// Validate the state was saved.
$config = $DB->get_field(RLIP_SCHEDULE_TABLE, 'config', array('id' => 1));
$this->assertRegExp('/s:5:"state";/', $config);
// Create a duplicate job.
$taskid2 = rlip_schedule_add_job($data);
// Get the initial duplicate job lastruntime and nextruntime values.
$initlastruntime = $DB->get_field('local_eliscore_sched_tasks', 'lastruntime', array('id' => $taskid2));
$initnextruntime = $DB->get_field('local_eliscore_sched_tasks', 'nextruntime', array('id' => $taskid2));
// Emulate the ELIS cron adjusting the job run times.
$task = $DB->get_record('local_eliscore_sched_tasks', array('id' => $taskid2));
$task->lastruntime = time();
$nextruntime = cron_next_run_time($task->lastruntime, (array) $task);
$task->nextruntime = $nextruntime;
$DB->update_record('local_eliscore_sched_tasks', $task);
// Attempt to do another import run.
$taskname2 = $DB->get_field('local_eliscore_sched_tasks', 'taskname', array('id' => $taskid2));
$result2 = run_ipjob($taskname2);
// Validate that the second import run attempt fails.
$this->assertEquals(false, $result2);
// Get the later lastruntime and nextruntime values.
$lastruntime = $DB->get_field('local_eliscore_sched_tasks', 'lastruntime', array('id' => $taskid2));
$nextruntime = $DB->get_field('local_eliscore_sched_tasks', 'nextruntime', array('id' => $taskid2));
// Validate that the job run time values are back to initial values.
$this->assertEquals($initlastruntime, $lastruntime);
$this->assertEquals($initnextruntime, $nextruntime);
}
/**
* Validate that, if the setting for disabling the internal Moodle cron is
* enabled, tasks do not run in the internal cron
*/
public function test_runipjobskipsjobsifdisabledincronenabled()
{
global $DB;
// Set up the export file name.
set_config('export_file', 'export.csv', 'dhexport_version1');
// Disable running in the standard cron.
set_config('disableincron', '1', 'local_datahub');
// Create the job (doesn't really matter which plugin).
$data = array('plugin' => 'dhexport_version1', 'period' => '5m', 'type' => 'dhexport');
$taskid = rlip_schedule_add_job($data);
// Set up the job to run on the next "cron".
$DB->execute("UPDATE {local_eliscore_sched_tasks}\n SET nextruntime = ?", array(0));
$DB->execute("UPDATE {" . RLIP_SCHEDULE_TABLE . "}\n SET nextruntime = ?", array(0));
// Run the export.
$taskname = $DB->get_field('local_eliscore_sched_tasks', 'taskname', array('id' => $taskid));
run_ipjob($taskname);
// Validate that tasks are not run by checking that their next runtime values have not been changed..
$exists = $DB->record_exists('local_eliscore_sched_tasks', array('nextruntime' => 0));
$this->assertTrue($exists);
$exists = $DB->record_exists(RLIP_SCHEDULE_TABLE, array('nextruntime' => 0));
$this->assertTrue($exists);
}
/**
* Validate that, when the available runtime is exceeded, IP leaves the next
* runtime for this plugin unchanged
*/
public function test_exportleavesnextruntimewhentimelimitexceeded()
{
global $CFG, $DB;
require_once $CFG->dirroot . '/local/datahub/lib.php';
// Load data.
$this->load_csv_data();
// Set up configuration.
set_config('export_path', '/datahub/dhexport_version1elis', 'dhexport_version1elis');
set_config('export_file', 'export_version1elis.csv', 'dhexport_version1elis');
set_config('nonincremental', 1, 'dhexport_version1elis');
// Create the job.
$data = array('plugin' => 'dhexport_version1elis', 'period' => '5m', 'type' => 'dhexport');
$taskid = rlip_schedule_add_job($data);
// Set next runtime values to a known state.
$DB->execute("UPDATE {local_eliscore_sched_tasks} SET nextruntime = ?", array(1));
$DB->execute("UPDATE {" . RLIP_SCHEDULE_TABLE . "} SET nextruntime = ?", array(1));
// Run the job.
$taskname = $DB->get_field('local_eliscore_sched_tasks', 'taskname', array('id' => $taskid));
run_ipjob($taskname, -1);
// Validate that nextruntime values haven't changed.
$exists = $DB->record_exists('local_eliscore_sched_tasks', array('nextruntime' => 1));
$this->assertTrue($exists);
$exists = $DB->record_exists(RLIP_SCHEDULE_TABLE, array('nextruntime' => 1));
$this->assertTrue($exists);
}
/**
* Validate that database logging works as specified for scheduled export
* tasks
*/
public function test_version1dbloggingsetsallfieldsduringscheduledrun()
{
global $CFG, $DB, $USER;
require_once $CFG->dirroot . '/local/datahub/lib.php';
// Set up the export file path.
$filename = $CFG->dataroot . '/rliptestexport.csv';
set_config('export_file', $filename, 'dhexport_version1');
// Set up data for one course and one enroled user.
$this->load_csv_data();
// Create a scheduled job.
$data = array('plugin' => 'dhexport_version1', 'period' => '5m', 'label' => 'bogus', 'type' => 'dhexport');
$taskid = rlip_schedule_add_job($data);
// Change the next runtime to a known value in the past.
$task = new stdClass();
$task->id = $taskid;
$task->nextruntime = 99;
$DB->update_record('local_eliscore_sched_tasks', $task);
$job = new stdClass();
$job->id = $DB->get_field(RLIP_SCHEDULE_TABLE, 'id', array('plugin' => 'dhexport_version1'));
$job->nextruntime = 99;
$DB->update_record(RLIP_SCHEDULE_TABLE, $job);
// Lower bound on starttime.
$starttime = time();
// Run the export.
$taskname = $DB->get_field('local_eliscore_sched_tasks', 'taskname', array('id' => $taskid));
run_ipjob($taskname);
// Upper bound on endtime.
$endtime = time();
// Data validation.
$select = "export = :export AND\n plugin = :plugin AND\n userid = :userid AND\n targetstarttime = :targetstarttime AND\n starttime >= :starttime AND\n endtime <= :endtime AND\n endtime >= starttime AND\n filesuccesses = :filesuccesses AND\n filefailures = :filefailures AND\n storedsuccesses = :storedsuccesses AND\n storedfailures = :storedfailures AND\n {$DB->sql_compare_text('statusmessage')} = :statusmessage AND\n dbops = :dbops AND\n unmetdependency = :unmetdependency";
$datestr = date('M_j_Y_His', $starttime);
$params = array('export' => 1, 'plugin' => 'dhexport_version1', 'userid' => $USER->id, 'targetstarttime' => 99, 'starttime' => $starttime, 'endtime' => $endtime, 'filesuccesses' => 0, 'filefailures' => 0, 'storedsuccesses' => 0, 'storedfailures' => 0, 'statusmessage' => 'Export file rliptestexport_' . $datestr . '.csv successfully created.', 'dbops' => -1, 'unmetdependency' => 0);
$exists = $DB->record_exists_select(RLIP_LOG_TABLE, $select, $params);
$this->assertTrue($exists);
}
function display_form()
{
global $DB;
$this->get_base_page_params();
$id = $this->optional_param('id', 0, PARAM_INT);
$target = $this->get_new_page(array('action' => 'form', 'plugin' => $this->get_ip_plugin()));
$params = array('plugin' => $this->get_ip_plugin(), 'type' => $this->type, 'name' => $this->name, 'id' => $id);
if (!empty($id) && ($ipjob = $DB->get_record(RLIP_SCHEDULE_TABLE, array('id' => $id)))) {
$data = unserialize($ipjob->config);
unset($data['id']);
$params = array_merge($params, $data);
}
$form = $this->type == 'dhimport' ? new $this->import_form_class($target->url, $params) : new $this->export_form_class($target->url, $params);
$form->set_data($params);
if ($form->is_cancelled()) {
$this->display_default();
return;
}
$data = $form->get_data();
if ($data && rlip_schedule_add_job((array) $data) > 0) {
$target = $this->get_new_page(array('plugin' => $this->get_ip_plugin()));
redirect($target->url);
} else {
$form->display();
}
}
/**
* Validate that, on subsequent runs, the export contains the header and only
* the more recent data row
*/
public function test_exportcontainsalldatasincelastrun()
{
global $CFG, $DB;
require_once $CFG->dirroot . '/local/datahub/lib.php';
// Load data.
$this->load_csv_data();
// Set up configuration.
set_config('export_path', '/datahub/dhexport_version1elis', 'dhexport_version1elis');
set_config('export_file', 'export_version1elis.csv', 'dhexport_version1elis');
set_config('nonincremental', 0, 'dhexport_version1elis');
// Create the job.
$data = array('plugin' => 'dhexport_version1elis', 'period' => '5m', 'type' => 'dhexport');
$taskid = rlip_schedule_add_job($data);
// Mark the job as having been run after the first record was created.
$DB->execute("UPDATE {local_eliscore_sched_tasks} SET lastruntime = ?", array(1000));
$DB->execute("UPDATE {" . RLIP_SCHEDULE_TABLE . "} SET lastruntime = ?", array(1000));
// Run the job.
$taskname = $DB->get_field('local_eliscore_sched_tasks', 'taskname', array('id' => $taskid));
$starttime = time();
run_ipjob($taskname);
// Path to export file.
$datestr = date('M_j_Y_His', $starttime);
$filepath = $CFG->dataroot . '/datahub/dhexport_version1elis/export_version1elis_' . $datestr . '.csv';
// Validate that the file exists.
$this->assertTrue(file_exists($filepath));
$handle = fopen($filepath, 'r');
$this->assertNotEquals(false, $handle);
// Validate header.
$header = fgetcsv($handle);
$this->assertEquals($this->get_header(), $header);
// Validate second row (i.e. earlier row skipped).
$newrecord = fgetcsv($handle);
$this->assertEquals($this->get_second_row(), $newrecord);
// Validate end of file.
$eof = fgetcsv($handle);
$this->assertEquals(false, $eof);
}
/**
* Validate that a scheduled import log file exists with the proper name
*/
public function test_version1importlogscheduled()
{
global $CFG, $DB, $USER;
require_once $CFG->dirroot . '/local/datahub/lib/rlip_importprovider_moodlefile.class.php';
$DB->delete_records('local_eliscore_sched_tasks');
$DB->delete_records(RLIP_SCHEDULE_TABLE);
// Set the file path to the dataroot.
$filepath = $CFG->dataroot . RLIP_DEFAULT_LOG_PATH;
// File path and name.
$filename = 'userscheduledimport.csv';
// File WILL BE DELETED after import so must copy to moodledata area.
// Note: file_path now relative to moodledata ($CFG->dataroot).
$filepath = '/local_datahub_phpunit/';
$testdir = $CFG->dataroot . $filepath;
@mkdir($testdir, 0777, true);
@copy(dirname(__FILE__) . "/fixtures/{$filename}", $testdir . $filename);
// Create a scheduled job.
$data = array('plugin' => 'dhimport_version1', 'period' => '5m', 'label' => 'bogus', 'type' => 'dhimport', 'userid' => $USER->id);
$taskid = rlip_schedule_add_job($data);
// Lower bound on starttime.
$starttime = time() - 100;
// Change the next runtime to a day from now.
$task = new stdClass();
$task->id = $taskid;
$task->nextruntime = $starttime + DAYSECS;
// Tomorrow?
$DB->update_record('local_eliscore_sched_tasks', $task);
$job = new stdClass();
$job->id = $DB->get_field(RLIP_SCHEDULE_TABLE, 'id', array('plugin' => 'dhimport_version1'));
$job->nextruntime = $starttime + DAYSECS;
// Tomorrow?
$DB->update_record(RLIP_SCHEDULE_TABLE, $job);
// Set up config for plugin so the scheduler knows about our csv file.
set_config('schedule_files_path', $filepath, 'dhimport_version1');
set_config('user_schedule_file', $filename, 'dhimport_version1');
// Run the import.
$taskname = $DB->get_field('local_eliscore_sched_tasks', 'taskname', array('id' => $taskid));
run_ipjob($taskname);
// Get timestamp from summary log.
$records = $DB->get_records(RLIP_LOG_TABLE, null, 'starttime DESC');
foreach ($records as $record) {
$starttime = $record->starttime;
break;
}
$format = get_string('logfile_timestamp', 'local_datahub');
$plugintype = 'import';
$plugin = 'dhimport_version1';
$manual = true;
$entity = 'user';
$testfilename = $plugintype . '_version1_scheduled_' . $entity . '_' . userdate($starttime, $format) . '.log';
$testfilename = self::get_current_logfile($CFG->dataroot . '/datahub/log/' . $testfilename);
$exists = file_exists($testfilename);
$this->assertTrue($exists);
// Cleanup test directory & import data file.
@unlink($testdir . $filename);
@rmdir($testdir);
}
/**
* Validate that SCHEDULED import obeys maxruntime
*
* @dataProvider fileandlinecountprovider
* @param string $filename The name of the file we are importing
* @param string $entity The entity type, such as 'user'
* @param int $numlines The total number of lines in the file
*/
public function test_scheduledimportobeysmaxruntime($filename, $entity, $numlines)
{
global $CFG, $DB;
require_once $CFG->dirroot . '/local/datahub/lib.php';
$filepath = '/local_datahub_phpunit/';
$filename = $filename;
set_config('schedule_files_path', $filepath, 'dhimport_version1elis');
set_config($entity . '_schedule_file', $filename, 'dhimport_version1elis');
// Set up the test directory.
$testdir = $CFG->dataroot . $filepath;
@mkdir($testdir, 0777, true);
@copy(dirname(__FILE__) . "/fixtures/{$filename}", $testdir . $filename);
// Create the job.
$data = array('plugin' => 'dhimport_version1elis', 'period' => '5m', 'type' => 'dhimport');
$taskid = rlip_schedule_add_job($data);
// Set next runtime values to a known state.
$DB->execute("UPDATE {local_eliscore_sched_tasks} SET nextruntime = ?", array(1));
$DB->execute("UPDATE {" . RLIP_SCHEDULE_TABLE . "} SET nextruntime = ?", array(1));
// Run the import.
$taskname = $DB->get_field('local_eliscore_sched_tasks', 'taskname', array('id' => $taskid));
$mintime = time();
run_ipjob($taskname, -1);
$maxtime = time();
// Clean-up data file & test dir.
@unlink($testdir . $filename);
@rmdir($testdir);
// Validation.
$params = array('export' => 0, 'plugin' => 'dhimport_version1elis', 'targetstarttime' => 1, 'filesuccesses' => 0, 'filefailures' => 0, 'storedsuccesses' => 0, 'storedfailures' => 0);
$exists = $DB->record_exists(RLIP_LOG_TABLE, $params);
$log = $DB->get_record(RLIP_LOG_TABLE, array('id' => 1));
// Validate entity type.
$this->assertEquals($entity, $log->entitytype);
// Validate status message.
$a = new stdClass();
$a->filename = $filename;
$a->recordsprocessed = 0;
$a->totalrecords = $numlines;
$expectedmessage = get_string('dblogimportexceedstimelimit', 'local_datahub', $a);
$this->assertEquals($expectedmessage, $log->statusmessage);
// Validate logged start time.
$this->assertGreaterThanOrEqual($mintime, $log->starttime);
$this->assertLessThanOrEqual($maxtime, $log->starttime);
// Validate logged end time.
$this->assertGreaterThanOrEqual($mintime, $log->endtime);
$this->assertLessThanOrEqual($maxtime, $log->endtime);
}
/**
* Validate that database logging works as specified for scheduled import
* tasks
*/
public function test_version1dbloggingsetsallfieldsduringscheduledimportrun()
{
global $CFG, $DB, $USER;
require_once $CFG->dirroot . '/local/datahub/lib/rlip_importprovider_moodlefile.class.php';
require_once $CFG->dirroot . '/local/datahub/lib.php';
$DB->delete_records('local_eliscore_sched_tasks');
$DB->delete_records(RLIP_SCHEDULE_TABLE);
// Set the file path to a fixed value.
$filepath = $CFG->dataroot;
// Store it at the system context.
$context = context_system::instance();
// File path and name.
$filename = 'userscheduledimport.csv';
// File WILL BE DELETED after import so must copy to moodledata area
// Note: file_path now relative to moodledata ($CFG->dataroot).
$filepath = '/local_datahub_phpunit/';
$testdir = $CFG->dataroot . $filepath;
@mkdir($testdir, 0777, true);
@copy(dirname(__FILE__) . "/fixtures/{$filename}", $testdir . $filename);
// Create a scheduled job.
$data = array('plugin' => 'dhimport_version1', 'period' => '5m', 'label' => 'bogus', 'type' => 'dhimport');
$taskid = rlip_schedule_add_job($data);
// Change the next runtime to a known value in the past.
$task = new stdClass();
$task->id = $taskid;
$task->nextruntime = 99;
$DB->update_record('local_eliscore_sched_tasks', $task);
$job = new stdClass();
$job->id = $DB->get_field(RLIP_SCHEDULE_TABLE, 'id', array('plugin' => 'dhimport_version1'));
$job->nextruntime = 99;
$DB->update_record(RLIP_SCHEDULE_TABLE, $job);
// Lower bound on starttime.
$starttime = time();
// Set up config for plugin so the scheduler knows about our csv file.
set_config('schedule_files_path', $filepath, 'dhimport_version1');
set_config('user_schedule_file', $filename, 'dhimport_version1');
// Run the import.
$taskname = $DB->get_field('local_eliscore_sched_tasks', 'taskname', array('id' => $taskid));
run_ipjob($taskname);
$message = 'One or more lines from import file userscheduledimport.csv failed because they contain data errors. ';
$message .= 'Please fix the import file and re-upload it.';
// Upper bound on endtime.
$endtime = time();
// Condition for the logpath.
$like = $DB->sql_like('logpath', ':logpath');
// Data validation.
$select = "export = :export AND\n plugin = :plugin AND\n userid = :userid AND\n targetstarttime = :targetstarttime AND\n starttime >= :starttime AND\n endtime <= :endtime AND\n endtime >= starttime AND\n filesuccesses = :filesuccesses AND\n filefailures = :filefailures AND\n storedsuccesses = :storedsuccesses AND\n storedfailures = :storedfailures AND\n {$DB->sql_compare_text('statusmessage')} = :statusmessage AND\n dbops = :dbops AND\n unmetdependency = :unmetdependency AND\n {$like} AND\n entitytype = :entitytype";
$params = array('export' => 0, 'plugin' => 'dhimport_version1', 'userid' => $USER->id, 'targetstarttime' => 99, 'starttime' => $starttime, 'endtime' => $endtime, 'filesuccesses' => 2, 'filefailures' => 2, 'storedsuccesses' => 0, 'storedfailures' => 0, 'statusmessage' => $message, 'dbops' => -1, 'unmetdependency' => 0, 'logpath' => "{$CFG->dataroot}/%", 'entitytype' => 'user');
$exists = $DB->record_exists_select(RLIP_LOG_TABLE, $select, $params);
$this->assertTrue($exists);
// Verify completed import deletes input csv file.
$this->assertFalse(file_exists($testdir . $filename));
// Clean-up data file & test dir.
@unlink($testdir . $filename);
@rmdir($testdir);
}