public function execute()
{
global $CFG, $DB;
require_once $CFG->libdir . '/csvlib.class.php';
require_once $CFG->libdir . '/moodlelib.php';
$csvfilepath = $this->arguments[0];
if ($csvfilepath[0] != '/') {
$csvfilepath = $this->cwd . DIRECTORY_SEPARATOR . $csvfilepath;
}
$iid = \csv_import_reader::get_new_iid('userprofile');
$type = 'userprofile';
$csvreader = new \csv_import_reader($iid, $type);
if (false === ($csvfile = file_get_contents($csvfilepath))) {
cli_error('Unable to load csv file. ' . error_get_last()['message']);
}
if (!$csvreader->load_csv_content($csvfile, 'utf-8', 'comma')) {
cli_error('Unalbe to parse csv file. ' . $csvreader->get_error());
}
if (!$csvreader->init()) {
cli_error('Unable to initialise csv reading');
}
$columns = $csvreader->get_columns();
$columnsids = array_flip($columns);
while (false !== ($row = $csvreader->next())) {
$category = $this->get_or_create_category($row[$columnsids['categoryname']], $row[$columnsids['categorysortorder']]);
$userfield = new \stdClass();
$userfield->shortname = $row[$columnsids['shortname']];
$userfield->name = $row[$columnsids['name']];
$userfield->datatype = $row[$columnsids['datatype']];
$userfield->description = $row[$columnsids['description']];
$userfield->descriptionformat = $row[$columnsids['descriptionformat']];
$userfield->categoryid = $category->id;
$userfield->sortorder = $row[$columnsids['sortorder']];
$userfield->required = $row[$columnsids['required']];
$userfield->locked = $row[$columnsids['locked']];
$userfield->visible = $row[$columnsids['visible']];
$userfield->forceunique = $row[$columnsids['forceunique']];
$userfield->signup = $row[$columnsids['signup']];
$userfield->defaultdata = $row[$columnsids['defaultdata']];
$userfield->defaultdataformat = $row[$columnsids['defaultdataformat']];
$userfield->param1 = $row[$columnsids['param1']];
$userfield->param2 = $row[$columnsids['param2']];
$userfield->param3 = $row[$columnsids['param3']];
$userfield->param4 = $row[$columnsids['param4']];
$userfield->param5 = $row[$columnsids['param5']];
$this->get_or_create_userfield($userfield);
}
}
/**
* Load CSV content for previewing.
*
* @param string $text The grade data being imported.
* @param string $encoding The type of encoding the file uses.
* @param string $separator The separator being used to define each field.
* @param int $previewrows How many rows are being previewed.
*/
public function load_csv_content($text, $encoding, $separator, $previewrows)
{
$this->raise_limits();
$this->iid = csv_import_reader::get_new_iid('grade');
$csvimport = new csv_import_reader($this->iid, 'grade');
$csvimport->load_csv_content($text, $encoding, $separator);
$this->error = $csvimport->get_error();
// Get header (field names).
$this->headers = $csvimport->get_columns();
$this->trim_headers();
$csvimport->init();
$this->previewdata = array();
for ($numlines = 0; $numlines <= $previewrows; $numlines++) {
$lines = $csvimport->next();
if ($lines) {
$this->previewdata[] = $lines;
}
}
}
$profilefieldname = 'profile_field_' . $proffield->shortname;
$PRF_FIELDS[] = $profilefieldname;
// Re-index $proffields with key as shortname. This will be
// used while checking if profile data is key and needs to be converted (eg. menu profile field)
$proffields[$profilefieldname] = $proffield;
unset($proffields[$key]);
}
}
if (empty($iid)) {
$mform1 = new admin_uploaduser_form1();
if ($formdata = $mform1->get_data()) {
$iid = csv_import_reader::get_new_iid('uploaduser');
$cir = new csv_import_reader($iid, 'uploaduser');
$content = $mform1->get_file_content('userfile');
$readcount = $cir->load_csv_content($content, $formdata->encoding, $formdata->delimiter_name);
$csvloaderror = $cir->get_error();
unset($content);
if (!is_null($csvloaderror)) {
print_error('csvloaderror', '', $returnurl, $csvloaderror);
}
// test if columns ok
$filecolumns = uu_validate_user_upload_columns($cir, $STD_FIELDS, $PRF_FIELDS, $returnurl);
// continue to form2
} else {
echo $OUTPUT->header();
echo $OUTPUT->heading_with_help(get_string('uploadusers', 'tool_uploaduser'), 'uploadusers', 'tool_uploaduser');
$mform1->display();
echo $OUTPUT->footer();
die;
}
} else {
if ($prof_fields = get_records('user_info_field')) {
foreach ($prof_fields as $prof_field) {
$PRF_FIELDS[] = 'profile_field_' . $prof_field->shortname;
}
unset($prof_fields);
}
if (empty($iid)) {
$mform = new admin_uploaduser_form1();
if ($formdata = $mform->get_data()) {
$iid = csv_import_reader::get_new_iid('uploaduser');
$cir = new csv_import_reader($iid, 'uploaduser');
$content = $mform->get_file_content('userfile');
$readcount = $cir->load_csv_content($content, $formdata->encoding, $formdata->delimiter_name, 'validate_user_upload_columns');
unset($content);
if ($readcount === false) {
error($cir->get_error(), $returnurl);
} else {
if ($readcount == 0) {
print_error('csvemptyfile', 'error', $returnurl);
}
}
// continue to form2
} else {
admin_externalpage_print_header();
print_heading_with_help(get_string('uploadusers'), 'uploadusers2');
$mform->display();
admin_externalpage_print_footer();
die;
}
} else {
$cir = new csv_import_reader($iid, 'uploaduser');
require_once $CFG->libdir . '/coursecatlib.php';
require_once $CFG->libdir . '/csvlib.class.php';
admin_externalpage_setup('tooluploadcourse');
$importid = optional_param('importid', '', PARAM_INT);
$previewrows = optional_param('previewrows', 10, PARAM_INT);
$returnurl = new moodle_url('/admin/tool/uploadcourse/index.php');
if (empty($importid)) {
$mform1 = new tool_uploadcourse_step1_form();
if ($form1data = $mform1->get_data()) {
$importid = csv_import_reader::get_new_iid('uploadcourse');
$cir = new csv_import_reader($importid, 'uploadcourse');
$content = $mform1->get_file_content('coursefile');
$readcount = $cir->load_csv_content($content, $form1data->encoding, $form1data->delimiter_name);
unset($content);
if ($readcount === false) {
print_error('csvfileerror', 'tool_uploadcourse', $returnurl, $cir->get_error());
} else {
if ($readcount == 0) {
print_error('csvemptyfile', 'error', $returnurl, $cir->get_error());
}
}
} else {
echo $OUTPUT->header();
echo $OUTPUT->heading_with_help(get_string('uploadcourses', 'tool_uploadcourse'), 'uploadcourses', 'tool_uploadcourse');
$mform1->display();
echo $OUTPUT->footer();
die;
}
} else {
$cir = new csv_import_reader($importid, 'uploadcourse');
}
// show the initial submission form.
if (!$iid) {
// If the import form has been submitted.
if ($formdata = $mform->get_data()) {
// Large files are likely to take their time and memory. Let PHP know
// that we'll take longer, and that the process should be recycled soon
// to free up memory.
core_php_time_limit::raise();
raise_memory_limit(MEMORY_EXTRA);
// Use current (non-conflicting) time stamp.
$importcode = get_new_importcode();
$text = $mform->get_file_content('userfile');
$iid = csv_import_reader::get_new_iid('grade');
$csvimport = new csv_import_reader($iid, 'grade');
$csvimport->load_csv_content($text, $formdata->encoding, $separator);
if ($error = $csvimport->get_error()) {
echo $OUTPUT->notification($error);
echo $OUTPUT->footer();
die;
}
// --- get header (field names) ---
$header = $csvimport->get_columns();
// Print a preview of the data.
$numlines = 0;
// 0 lines previewed so far.
echo $OUTPUT->heading(get_string('importpreview', 'grades'));
foreach ($header as $i => $h) {
$h = trim($h);
// Remove whitespace.
$h = clean_param($h, PARAM_RAW);
// Clean the header.
public function test_csv_functions()
{
global $CFG;
$csvexport = new csv_export_writer();
$csvexport->set_filename('unittest');
foreach ($this->testdata as $data) {
$csvexport->add_data($data);
}
$csvoutput = $csvexport->print_csv_data(true);
$this->assertEquals($csvoutput, $this->teststring);
$test_data = csv_export_writer::print_array($this->testdata, 'comma', '"', true);
$this->assertEquals($test_data, $this->teststring);
// Testing that the content is imported correctly.
$iid = csv_import_reader::get_new_iid('lib');
$csvimport = new csv_import_reader($iid, 'lib');
$contentcount = $csvimport->load_csv_content($this->teststring, 'utf-8', 'comma');
$csvimport->init();
$dataset = array();
$dataset[] = $csvimport->get_columns();
while ($record = $csvimport->next()) {
$dataset[] = $record;
}
$csvimport->cleanup();
$csvimport->close();
$this->assertEquals($dataset, $this->testdata);
// Testing for the wrong count of columns.
$errortext = get_string('csvweirdcolumns', 'error');
$iid = csv_import_reader::get_new_iid('lib');
$csvimport = new csv_import_reader($iid, 'lib');
$contentcount = $csvimport->load_csv_content($this->teststring2, 'utf-8', 'comma');
$importerror = $csvimport->get_error();
$csvimport->cleanup();
$csvimport->close();
$this->assertEquals($importerror, $errortext);
// Testing for empty content
$errortext = get_string('csvemptyfile', 'error');
$iid = csv_import_reader::get_new_iid('lib');
$csvimport = new csv_import_reader($iid, 'lib');
$contentcount = $csvimport->load_csv_content($this->teststring3, 'utf-8', 'comma');
$importerror = $csvimport->get_error();
$csvimport->cleanup();
$csvimport->close();
$this->assertEquals($importerror, $errortext);
// Testing for a tab separated file.
// The tab separated file has a trailing tab and extra blank lines at the end of the file.
$filename = $CFG->dirroot . '/lib/tests/fixtures/tabfile.csv';
$fp = fopen($filename, 'r');
$tabdata = fread($fp, filesize($filename));
fclose($fp);
$iid = csv_import_reader::get_new_iid('tab');
$csvimport = new csv_import_reader($iid, 'tab');
$contentcount = $csvimport->load_csv_content($tabdata, 'utf-8', 'tab');
// This should import four rows including the headings.
$this->assertEquals($contentcount, 4);
// Testing for empty lines.
$iid = csv_import_reader::get_new_iid('blanklines');
$csvimport = new csv_import_reader($iid, 'blanklines');
$contentcount = $csvimport->load_csv_content($this->teststring4, 'utf-8', 'comma');
// Five lines including the headings should be imported.
$this->assertEquals($contentcount, 5);
}
}
if (!file_exists($options['file'])) {
echo get_string('invalidcsvfile', 'tool_uploadusercli') . "\n";
echo $help;
die;
}
// Encoding.
$encodings = core_text::get_encodings();
if (!isset($encodings[$options['encoding']])) {
echo get_string('invalidencoding', 'tool_uploadusercli') . "\n";
echo $help;
die;
}
// Emulate admin session.
cron_setup_user();
// Let's get started!
$content = file_get_contents($options['file']);
$importid = csv_import_reader::get_new_iid('uploaduser');
$cir = new csv_import_reader($importid, 'uploaduser');
$readcount = $cir->load_csv_content($content, $options['encoding'], $options['delimiter']);
if ($readcount === false) {
print_error('csvfileerror', 'tool_uploadusercli', '', $cir->get_error());
} else {
if ($readcount == 0) {
print_error('csvemptyfile', 'error', '', $cir->get_error());
}
}
unset($content);
$processor = new tool_uploadusercli_processor($cir, $processoroptions);
$processor->execute();
print "Done.\n";
/**
* @param array $options associative delimiter,enclosure,encoding,updateexisting,settings
*/
public function process_csv($data, $csvcontent, $options = null)
{
global $CFG, $DB;
require_once "{$CFG->libdir}/csvlib.class.php";
@set_time_limit(0);
raise_memory_limit(MEMORY_EXTRA);
$iid = \csv_import_reader::get_new_iid('moddataform');
$cir = new \csv_import_reader($iid, 'moddataform');
$delimiter = !empty($options['delimiter']) ? $options['delimiter'] : $this->_delimiter;
$enclosure = !empty($options['enclosure']) ? $options['enclosure'] : $this->_enclosure;
$encoding = !empty($options['encoding']) ? $options['encoding'] : $this->_encoding;
$fieldsettings = !empty($options['settings']) ? $options['settings'] : array();
$readcount = $cir->load_csv_content($csvcontent, $encoding, $delimiter);
if (empty($readcount)) {
$data->errors[] = $cir->get_error();
return $data;
}
// Csv column headers.
if (!($fieldnames = $cir->get_columns())) {
$data->errors[] = $cir->get_error();
return $data;
}
// Are we updating existing entries?
$existingkeys = array();
$keyname = null;
if ($updateexisting = !empty($options['updateexisting'])) {
if (isset($fieldnames['entryid'])) {
$keyname = 'entryid';
} else {
$keyname = reset($fieldnames);
if ($field = $this->df->field_manager->get_field_by_name($keyname)) {
$params = array('fieldid' => $field->id);
$existingkeys = $DB->get_records_menu('dataform_contents', $params, '', 'entryid,content');
}
}
}
// Are we adding the imported entries to every participant?
$addperparticipant = (!empty($options['addperparticipant']) and $users = $this->df->grade_manager->get_gradebook_users());
$i = 0;
$cir->init();
while ($csvrecord = $cir->next()) {
$csvrecord = array_combine($fieldnames, $csvrecord);
// Add the entry for every participant.
if ($addperparticipant) {
foreach ($users as $userid => $unused) {
// Set the entry id.
$i++;
$entryid = -$i;
$data->eids[$entryid] = $entryid;
$data->{"entry_{$entryid}_userid"} = $userid;
// Iterate the fields and collate their entry content.
foreach ($fieldsettings as $fieldid => $importsettings) {
$field = $this->df->field_manager->get_field_by_id($fieldid);
$data = $field->prepare_import_content($data, $importsettings, $csvrecord, $entryid);
}
}
continue;
}
// Get the entry id.
$entryid = 0;
if ($updateexisting and $keyname) {
if ($keyname == 'entryid') {
if (!empty($csvrecord['entryid'])) {
$entryid = $csvrecord['entryid'];
}
} else {
if ($existingkeys and !empty($csvrecord[$keyname])) {
$entryid = array_search($csvrecord[$keyname], $existingkeys);
}
}
}
if (!$entryid) {
$i++;
$entryid = -$i;
}
$data->eids[$entryid] = $entryid;
// Iterate the fields and collate their entry content.
foreach ($fieldsettings as $fieldid => $importsettings) {
$field = $this->df->field_manager->get_field_by_id($fieldid);
$data = $field->prepare_import_content($data, $importsettings, $csvrecord, $entryid);
}
}
$cir->cleanup(true);
$cir->close();
return $data;
}
public function test_csv_functions() {
$csvexport = new csv_export_writer();
$csvexport->set_filename('unittest');
foreach ($this->testdata as $data) {
$csvexport->add_data($data);
}
$csvoutput = $csvexport->print_csv_data(true);
$this->assertEquals($csvoutput, $this->teststring);
$test_data = csv_export_writer::print_array($this->testdata, 'comma', '"', true);
$this->assertEquals($test_data, $this->teststring);
// Testing that the content is imported correctly.
$iid = csv_import_reader::get_new_iid('lib');
$csvimport = new csv_import_reader($iid, 'lib');
$contentcount = $csvimport->load_csv_content($this->teststring, 'utf-8', 'comma');
$csvimport->init();
$dataset = array();
$dataset[] = $csvimport->get_columns();
while ($record = $csvimport->next()) {
$dataset[] = $record;
}
$csvimport->cleanup();
$csvimport->close();
$this->assertEquals($dataset, $this->testdata);
// Testing for the wrong count of columns.
$errortext = get_string('csvweirdcolumns', 'error');
$iid = csv_import_reader::get_new_iid('lib');
$csvimport = new csv_import_reader($iid, 'lib');
$contentcount = $csvimport->load_csv_content($this->teststring2, 'utf-8', 'comma');
$importerror = $csvimport->get_error();
$csvimport->cleanup();
$csvimport->close();
$this->assertEquals($importerror, $errortext);
// Testing for empty content
$errortext = get_string('csvemptyfile', 'error');
$iid = csv_import_reader::get_new_iid('lib');
$csvimport = new csv_import_reader($iid, 'lib');
$contentcount = $csvimport->load_csv_content($this->teststring3, 'utf-8', 'comma');
$importerror = $csvimport->get_error();
$csvimport->cleanup();
$csvimport->close();
$this->assertEquals($importerror, $errortext);
}
/**
* @param stored_file $file
* @param string $encoding
* @param string $delimiter
* @param context $defaultcontext
* @return array
*/
protected function process_upload_file($file, $encoding, $delimiter, $defaultcontext)
{
global $CFG, $DB;
require_once $CFG->libdir . '/csvlib.class.php';
$cohorts = array(0 => array('errors' => array(), 'warnings' => array(), 'data' => array()));
// Read and parse the CSV file using csv library.
$content = $file->get_content();
if (!$content) {
$cohorts[0]['errors'][] = new lang_string('csvemptyfile', 'error');
return $cohorts;
}
$uploadid = csv_import_reader::get_new_iid('uploadcohort');
$cir = new csv_import_reader($uploadid, 'uploadcohort');
$readcount = $cir->load_csv_content($content, $encoding, $delimiter);
unset($content);
if (!$readcount) {
$cohorts[0]['errors'][] = get_string('csvloaderror', 'error', $cir->get_error());
return $cohorts;
}
$columns = $cir->get_columns();
// Check that columns include 'name' and warn about extra columns.
$allowedcolumns = array('contextid', 'name', 'idnumber', 'description', 'descriptionformat', 'visible');
$additionalcolumns = array('context', 'category', 'category_id', 'category_idnumber', 'category_path');
$displaycolumns = array();
$extracolumns = array();
$columnsmapping = array();
foreach ($columns as $i => $columnname) {
$columnnamelower = preg_replace('/ /', '', core_text::strtolower($columnname));
$columnsmapping[$i] = null;
if (in_array($columnnamelower, $allowedcolumns)) {
$displaycolumns[$columnnamelower] = $columnname;
$columnsmapping[$i] = $columnnamelower;
} else {
if (in_array($columnnamelower, $additionalcolumns)) {
$columnsmapping[$i] = $columnnamelower;
} else {
$extracolumns[] = $columnname;
}
}
}
if (!in_array('name', $columnsmapping)) {
$cohorts[0]['errors'][] = new lang_string('namecolumnmissing', 'cohort');
return $cohorts;
}
if ($extracolumns) {
$cohorts[0]['warnings'][] = new lang_string('csvextracolumns', 'cohort', s(join(', ', $extracolumns)));
}
if (!isset($displaycolumns['contextid'])) {
$displaycolumns['contextid'] = 'contextid';
}
$cohorts[0]['data'] = $displaycolumns;
// Parse data rows.
$cir->init();
$rownum = 0;
$idnumbers = array();
$haserrors = false;
$haswarnings = false;
while ($row = $cir->next()) {
$rownum++;
$cohorts[$rownum] = array('errors' => array(), 'warnings' => array(), 'data' => array());
$hash = array();
foreach ($row as $i => $value) {
if ($columnsmapping[$i]) {
$hash[$columnsmapping[$i]] = $value;
}
}
$this->clean_cohort_data($hash);
$warnings = $this->resolve_context($hash, $defaultcontext);
$cohorts[$rownum]['warnings'] = array_merge($cohorts[$rownum]['warnings'], $warnings);
if (!empty($hash['idnumber'])) {
if (isset($idnumbers[$hash['idnumber']]) || $DB->record_exists('cohort', array('idnumber' => $hash['idnumber']))) {
$cohorts[$rownum]['errors'][] = new lang_string('duplicateidnumber', 'cohort');
}
$idnumbers[$hash['idnumber']] = true;
}
if (empty($hash['name'])) {
$cohorts[$rownum]['errors'][] = new lang_string('namefieldempty', 'cohort');
}
$cohorts[$rownum]['data'] = array_intersect_key($hash, $cohorts[0]['data']);
$haserrors = $haserrors || !empty($cohorts[$rownum]['errors']);
$haswarnings = $haswarnings || !empty($cohorts[$rownum]['warnings']);
}
if ($haserrors) {
$cohorts[0]['errors'][] = new lang_string('csvcontainserrors', 'cohort');
}
if ($haswarnings) {
$cohorts[0]['warnings'][] = new lang_string('csvcontainswarnings', 'cohort');
}
return $cohorts;
}
$options['file'] = realpath($options['file']);
}
if (!file_exists($options['file'])) {
echo get_string('invalidcsvfile', 'tool_uploadcategory') . "\n";
echo $help;
die;
}
// Encoding.
$encodings = core_text::get_encodings();
if (!isset($encodings[$options['encoding']])) {
echo get_string('invalidencoding', 'tool_uploadcategory') . "\n";
echo $help;
die;
}
// Emulate admin session.
cron_setup_user();
// Let's get started!
$content = file_get_contents($options['file']);
$importid = csv_import_reader::get_new_iid('uploadcoursecategory');
$cir = new csv_import_reader($importid, 'uploadcoursecategory');
$readcount = $cir->load_csv_content($content, $options['encoding'], $options['delimiter']);
if ($readcount === false) {
print_error('csvfileerror', 'tool_uploadcoursecategory', '', $cir->get_error());
} else {
if ($readcount == 0) {
print_error('csvemptyfile', 'error', '', $cir->get_error());
}
}
unset($content);
$processor = new tool_uploadcoursecategory_processor($cir, $processoroptions);
$processor->execute();
