private function _getEncodingsArray() { return aEncodingsArray(); }
/** * This function import CSV file to responses table * * @param string $sFullFilePath * @param integer $iSurveyId * @param array $aOptions * Return array $result ("errors","warnings","success") */ function CSVImportResponses($sFullFilePath, $iSurveyId, $aOptions = array()) { $clang = Yii::app()->lang; // Default optional if (!isset($aOptions['bDeleteFistLine'])) { $aOptions['bDeleteFistLine'] = true; } // By default delete first line (vvimport) if (!isset($aOptions['sExistingId'])) { $aOptions['sExistingId'] = "ignore"; } // By default exclude existing id if (!isset($aOptions['bNotFinalized'])) { $aOptions['bNotFinalized'] = false; } // By default don't change finalized part if (!isset($aOptions['sCharset']) || !$aOptions['sCharset']) { $aOptions['sCharset'] = "utf8"; } if (!isset($aOptions['sSeparator'])) { $aOptions['sSeparator'] = "\t"; } if (!isset($aOptions['sQuoted'])) { $aOptions['sQuoted'] = "\""; } // Fix some part if (!array_key_exists($aOptions['sCharset'], aEncodingsArray())) { $aOptions['sCharset'] = "utf8"; } // Prepare an array of sentence for result $CSVImportResult = array(); // Read the file $handle = fopen($sFullFilePath, "r"); // Need to be adapted for Mac ? in options ? while (!feof($handle)) { $buffer = fgets($handle); //To allow for very long lines . Another option is fgetcsv (0 to length), but need mb_convert_encoding $aFileResponses[] = mb_convert_encoding($buffer, "UTF-8", $aOptions['sCharset']); } // Close the file fclose($handle); if ($aOptions['bDeleteFistLine']) { array_shift($aFileResponses); } $aRealFieldNames = Yii::app()->db->getSchema()->getTable(SurveyDynamic::model($iSurveyId)->tableName())->getColumnNames(); //$aCsvHeader=array_map("trim",explode($aOptions['sSeparator'], trim(array_shift($aFileResponses)))); $aCsvHeader = str_getcsv(array_shift($aFileResponses), $aOptions['sSeparator'], $aOptions['sQuoted']); $aLemFieldNames = LimeExpressionManager::getLEMqcode2sgqa($iSurveyId); $aKeyForFieldNames = array(); // An array assicated each fieldname with corresponding responses key if (!$aCsvHeader) { $CSVImportResult['errors'][] = $clang->gT("File seems empty or has only one line"); return $CSVImportResult; } // Assign fieldname with $aFileResponses[] key foreach ($aRealFieldNames as $sFieldName) { if (in_array($sFieldName, $aCsvHeader)) { // First pass : simple associated $aKeyForFieldNames[$sFieldName] = array_search($sFieldName, $aCsvHeader); } elseif (in_array($sFieldName, $aLemFieldNames)) { // Second pass : LEM associated $sLemFieldName = array_search($sFieldName, $aLemFieldNames); if (in_array($sLemFieldName, $aCsvHeader)) { $aKeyForFieldNames[$sFieldName] = array_search($sLemFieldName, $aCsvHeader); } elseif ($aOptions['bForceImport']) { // as fallback just map questions in order of apperance // find out where the answer data columns start in CSV if (!isset($csv_ans_start_index)) { foreach ($aCsvHeader as $i => $name) { if (preg_match('/^\\d+X\\d+X\\d+/', $name)) { $csv_ans_start_index = $i; break; } } } // find out where the answer data columns start in destination table if (!isset($table_ans_start_index)) { foreach ($aRealFieldNames as $i => $name) { if (preg_match('/^\\d+X\\d+X\\d+/', $name)) { $table_ans_start_index = $i; break; } } } // map answers in order if (isset($table_ans_start_index, $csv_ans_start_index)) { $csv_index = array_search($sFieldName, $aRealFieldNames) - $table_ans_start_index + $csv_ans_start_index; if ($csv_index < sizeof($aCsvHeader)) { $aKeyForFieldNames[$sFieldName] = $csv_index; } else { $force_import_failed = true; break; } } } } } // check if forced error failed if (isset($force_import_failed)) { $CSVImportResult['errors'][] = $clang->gT("Import failed: Forced import was requested but the input file doesn't contain enough columns to fill the survey."); return $CSVImportResult; } // make sure at least one answer was imported before commiting foreach ($aKeyForFieldNames as $field => $index) { if (preg_match('/^\\d+X\\d+X\\d+/', $field)) { $import_ok = true; break; } } if (!isset($import_ok)) { $CSVImportResult['errors'][] = $clang->gT("Import failed: No answers could be mapped."); return $CSVImportResult; } // Now it's time to import // Some var to return $iNbResponseLine = 0; $iNbResponseExisting = 0; $aResponsesInserted = array(); $aResponsesUpdated = array(); $aResponsesError = array(); $aExistingsId = array(); $iMaxId = 0; // If we set the id, keep the max // Some specific header (with options) $iIdKey = array_search('id', $aCsvHeader); // the id is allways needed and used a lot if (is_int($iIdKey)) { unset($aKeyForFieldNames['id']); } $iSubmitdateKey = array_search('submitdate', $aCsvHeader); // submitdate can be forced to null if (is_int($iSubmitdateKey)) { unset($aKeyForFieldNames['submitdate']); } $iIdReponsesKey = is_int($iIdKey) ? $iIdKey : 0; // The key for reponses id: id column or first column if not exist // Import each responses line here while ($sResponses = array_shift($aFileResponses)) { $iNbResponseLine++; $bExistingsId = false; $aResponses = str_getcsv($sResponses, $aOptions['sSeparator'], $aOptions['sQuoted']); if ($iIdKey !== false) { $oSurvey = SurveyDynamic::model($iSurveyId)->findByPk($aResponses[$iIdKey]); if ($oSurvey) { $bExistingsId = true; $aExistingsId[] = $aResponses[$iIdKey]; // Do according to option switch ($aOptions['sExistingId']) { case 'replace': SurveyDynamic::model($iSurveyId)->deleteByPk($aResponses[$iIdKey]); SurveyDynamic::sid($iSurveyId); $oSurvey = new SurveyDynamic(); break; case 'replaceanswers': break; case 'renumber': SurveyDynamic::sid($iSurveyId); $oSurvey = new SurveyDynamic(); break; case 'skip': case 'ignore': default: $oSurvey = false; // Remove existing survey : don't import again break; } } else { SurveyDynamic::sid($iSurveyId); $oSurvey = new SurveyDynamic(); } } else { SurveyDynamic::sid($iSurveyId); $oSurvey = new SurveyDynamic(); } if ($oSurvey) { // First rule for id and submitdate if (is_int($iIdKey)) { if (!$bExistingsId) { $oSurvey->id = $aResponses[$iIdKey]; $iMaxId = $aResponses[$iIdKey] > $iMaxId ? $aResponses[$iIdKey] : $iMaxId; } elseif ($aOptions['sExistingId'] == 'replace' || $aOptions['sExistingId'] == 'replaceanswers') { $oSurvey->id = $aResponses[$iIdKey]; } } if ($aOptions['bNotFinalized']) { $oSurvey->submitdate = new CDbExpression('NULL'); } elseif (is_int($iSubmitdateKey)) { if ($aResponses[$iSubmitdateKey] == '{question_not_shown}' || trim($aResponses[$iSubmitdateKey] == '')) { $oSurvey->submitdate = new CDbExpression('NULL'); } else { // Maybe control valid date : see http://php.net/manual/en/function.checkdate.php#78362 for example $oSurvey->submitdate = $aResponses[$iSubmitdateKey]; } } foreach ($aKeyForFieldNames as $sFieldName => $iFieldKey) { if ($aResponses[$iFieldKey] == '{question_not_shown}') { $oSurvey->{$sFieldName} = new CDbExpression('NULL'); } else { $sResponse = str_replace(array("{quote}", "{tab}", "{cr}", "{newline}", "{lbrace}"), array("\"", "\t", "\r", "\n", "{"), $aResponses[$iFieldKey]); $oSurvey->{$sFieldName} = $sResponse; } } // We use transaction to prevent DB error $oTransaction = Yii::app()->db->beginTransaction(); try { if (isset($oSurvey->id) && !is_null($oSurvey->id)) { switchMSSQLIdentityInsert('survey_' . $iSurveyId, true); $bSwitched = true; } if ($oSurvey->save()) { $oTransaction->commit(); if ($bExistingsId && $aOptions['sExistingId'] != 'renumber') { $aResponsesUpdated[] = $aResponses[$iIdReponsesKey]; } else { $aResponsesInserted[] = $aResponses[$iIdReponsesKey]; } } else { $oTransaction->rollBack(); $aResponsesError[] = $aResponses[$iIdReponsesKey]; } if (isset($bSwitched) && $bSwitched == true) { switchMSSQLIdentityInsert('survey_' . $iSurveyId, false); $bSwitched = false; } } catch (Exception $oException) { $oTransaction->rollBack(); $aResponsesError[] = $aResponses[$iIdReponsesKey]; // Show some error to user ? // $CSVImportResult['errors'][]=$oException->getMessage(); // Show it in view // tracevar($oException->getMessage());// Show it in console (if debug is set) } } } // Fix max next id (for pgsql) // mysql dot need fix, but what for mssql ? // Do a model function for this can be a good idea (see activate_helper/activateSurvey) if (Yii::app()->db->driverName == 'pgsql') { $sSequenceName = Yii::app()->db->getSchema()->getTable("{{survey_{$iSurveyId}}}")->sequenceName; $iActualSerial = Yii::app()->db->createCommand("SELECT last_value FROM {$sSequenceName}")->queryScalar(); if ($iActualSerial < $iMaxId) { $sQuery = "SELECT setval(pg_get_serial_sequence('{{survey_{$iSurveyId}}}', 'id'),{$iMaxId},false);"; $result = @Yii::app()->db->createCommand($sQuery)->execute(); } } // End of import // Construction of returned information if ($iNbResponseLine) { $CSVImportResult['success'][] = sprintf($clang->gT("%s response lines in your file."), $iNbResponseLine); } else { $CSVImportResult['errors'][] = $clang->gT("No response lines in your file."); } if (count($aResponsesInserted)) { $CSVImportResult['success'][] = sprintf($clang->gT("%s responses were inserted."), count($aResponsesInserted)); // Maybe add implode aResponsesInserted array } if (count($aResponsesUpdated)) { $CSVImportResult['success'][] = sprintf($clang->gT("%s responses were updated."), count($aResponsesUpdated)); } if (count($aResponsesError)) { $CSVImportResult['errors'][] = sprintf($clang->gT("%s responses cannot be inserted or updated."), count($aResponsesError)); } if (count($aExistingsId) && ($aOptions['sExistingId'] == 'skip' || $aOptions['sExistingId'] == 'ignore')) { $CSVImportResult['warnings'][] = sprintf($clang->gT("%s responses already exist."), count($aExistingsId)); } return $CSVImportResult; }
/** * import from csv */ function import($iSurveyId) { $clang = $this->getController()->lang; $iSurveyId = (int) $iSurveyId; if (!Permission::model()->hasSurveyPermission($iSurveyId, 'tokens', 'import')) { Yii::app()->session['flashmessage'] = $clang->gT("You do not have sufficient rights to access this page."); $this->getController()->redirect(array("/admin/survey/sa/view/surveyid/{$iSurveyId}")); } // CHECK TO SEE IF A TOKEN TABLE EXISTS FOR THIS SURVEY $bTokenExists = tableExists('{{tokens_' . $iSurveyId . '}}'); if (!$bTokenExists) { self::_newtokentable($iSurveyId); } App()->getClientScript()->registerScriptFile(Yii::app()->getConfig('adminscripts') . 'tokensimport.js'); $aEncodings = aEncodingsArray(); if (Yii::app()->request->getPost('submit')) { if (Yii::app()->request->getPost('csvcharset') && Yii::app()->request->getPost('csvcharset')) { $uploadcharset = Yii::app()->request->getPost('csvcharset'); if (!array_key_exists($uploadcharset, $aEncodings)) { $uploadcharset = 'auto'; } $filterduplicatetoken = Yii::app()->request->getPost('filterduplicatetoken') && Yii::app()->request->getPost('filterduplicatetoken') == 'on'; $filterblankemail = Yii::app()->request->getPost('filterblankemail') && Yii::app()->request->getPost('filterblankemail') == 'on'; } $attrfieldnames = getAttributeFieldNames($iSurveyId); $duplicatelist = array(); $invalidemaillist = array(); $invalidformatlist = array(); $firstline = array(); $sPath = Yii::app()->getConfig('tempdir'); $sFileTmpName = $_FILES['the_file']['tmp_name']; $sFilePath = $sPath . '/' . randomChars(20); if (!@move_uploaded_file($sFileTmpName, $sFilePath)) { $aData['sError'] = $clang->gT("Upload file not found. Check your permissions and path ({$sFilePath}) for the upload directory"); $aData['aEncodings'] = $aEncodings; $aData['iSurveyId'] = $aData['surveyid'] = $iSurveyId; $aData['thissurvey'] = getSurveyInfo($iSurveyId); $this->_renderWrappedTemplate('token', array('tokenbar', 'csvupload'), $aData); } else { $xz = 0; $recordcount = 0; $xv = 0; // This allows to read file with MAC line endings too @ini_set('auto_detect_line_endings', true); // open it and trim the ednings $tokenlistarray = file($sFilePath); $sBaseLanguage = Survey::model()->findByPk($iSurveyId)->language; if (!Yii::app()->request->getPost('filterduplicatefields') || Yii::app()->request->getPost('filterduplicatefields') && count(Yii::app()->request->getPost('filterduplicatefields')) == 0) { $filterduplicatefields = array('firstname', 'lastname', 'email'); } else { $filterduplicatefields = Yii::app()->request->getPost('filterduplicatefields'); } $separator = returnGlobal('separator'); foreach ($tokenlistarray as $buffer) { $buffer = @mb_convert_encoding($buffer, "UTF-8", $uploadcharset); if ($recordcount == 0) { // Parse first line (header) from CSV $buffer = removeBOM($buffer); // We alow all field except tid because this one is really not needed. $allowedfieldnames = array('participant_id', 'firstname', 'lastname', 'email', 'emailstatus', 'token', 'language', 'blacklisted', 'sent', 'remindersent', 'remindercount', 'validfrom', 'validuntil', 'completed', 'usesleft'); $allowedfieldnames = array_merge($attrfieldnames, $allowedfieldnames); // Some header don't have same column name $aReplacedFields = array('invited' => 'sent'); switch ($separator) { case 'comma': $separator = ','; break; case 'semicolon': $separator = ';'; break; default: $comma = substr_count($buffer, ','); $semicolon = substr_count($buffer, ';'); if ($semicolon > $comma) { $separator = ';'; } else { $separator = ','; } } $firstline = str_getcsv($buffer, $separator, '"'); $firstline = array_map('trim', $firstline); $ignoredcolumns = array(); // Now check the first line for invalid fields foreach ($firstline as $index => $fieldname) { $firstline[$index] = preg_replace("/(.*) <[^,]*>\$/", "\$1", $fieldname); $fieldname = $firstline[$index]; if (!in_array($fieldname, $allowedfieldnames)) { $ignoredcolumns[] = $fieldname; } if (array_key_exists($fieldname, $aReplacedFields)) { $firstline[$index] = $aReplacedFields[$fieldname]; } } if (!in_array('firstname', $firstline) || !in_array('lastname', $firstline) || !in_array('email', $firstline)) { $recordcount = count($tokenlistarray); break; } } else { $line = str_getcsv($buffer, $separator, '"'); if (count($firstline) != count($line)) { $invalidformatlist[] = $recordcount; $recordcount++; continue; } $writearray = array_combine($firstline, $line); //kick out ignored columns foreach ($ignoredcolumns as $column) { unset($writearray[$column]); } $dupfound = false; $invalidemail = false; if ($filterduplicatetoken != false) { $dupquery = "SELECT count(tid) from {{tokens_" . intval($iSurveyId) . "}} where 1=1"; foreach ($filterduplicatefields as $field) { if (isset($writearray[$field])) { $dupquery .= " and " . Yii::app()->db->quoteColumnName($field) . " = " . Yii::app()->db->quoteValue($writearray[$field]); } } $dupresult = Yii::app()->db->createCommand($dupquery)->queryScalar(); if ($dupresult > 0) { $dupfound = true; $duplicatelist[] = Yii::app()->db->quoteValue($writearray['firstname']) . " " . Yii::app()->db->quoteValue($writearray['lastname']) . " (" . Yii::app()->db->quoteValue($writearray['email']) . ")"; } } $writearray['email'] = trim($writearray['email']); //treat blank emails if ($filterblankemail && $writearray['email'] == '') { $invalidemail = true; $invalidemaillist[] = $line[0] . " " . $line[1] . " ( )"; } if ($writearray['email'] != '') { $aEmailAddresses = explode(';', $writearray['email']); foreach ($aEmailAddresses as $sEmailaddress) { if (!validateEmailAddress($sEmailaddress)) { $invalidemail = true; $invalidemaillist[] = $line[0] . " " . $line[1] . " (" . $line[2] . ")"; } } } if (isset($writearray['token'])) { $writearray['token'] = sanitize_token($writearray['token']); } if (!$dupfound && !$invalidemail) { // unset all empty value foreach ($writearray as $key => $value) { if ($writearray[$key] == "") { unset($writearray[$key]); } if (substr($value, 0, 1) == '"' && substr($value, -1) == '"') { // Fix CSV quote $value = substr($value, 1, -1); } } // Some default value : to be moved to Token model rules in future release ? // But think we have to accept invalid email etc ... then use specific scenario $writearray['emailstatus'] = isset($writearray['emailstatus']) ? $writearray['emailstatus'] : "OK"; $writearray['language'] = isset($writearray['language']) ? $writearray['language'] : $sBaseLanguage; $oToken = Token::create($iSurveyId); foreach ($writearray as $key => $value) { //if(in_array($key,$oToken->attributes)) Not needed because we filter attributes before $oToken->{$key} = $value; } $ir = $oToken->save(); if (!$ir) { $duplicatelist[] = $writearray['firstname'] . " " . $writearray['lastname'] . " (" . $writearray['email'] . ")"; } else { $xz++; } } $xv++; } $recordcount++; } $recordcount = $recordcount - 1; unlink($sFilePath); $aData['tokenlistarray'] = $tokenlistarray; $aData['xz'] = $xz; $aData['xv'] = $xv; $aData['recordcount'] = $recordcount; $aData['firstline'] = $firstline; $aData['duplicatelist'] = $duplicatelist; $aData['invalidformatlist'] = $invalidformatlist; $aData['invalidemaillist'] = $invalidemaillist; $aData['thissurvey'] = getSurveyInfo($iSurveyId); $aData['iSurveyId'] = $aData['surveyid'] = $iSurveyId; $this->_renderWrappedTemplate('token', array('tokenbar', 'csvpost'), $aData); } } else { $aData['aEncodings'] = $aEncodings; $aData['iSurveyId'] = $iSurveyId; $aData['thissurvey'] = getSurveyInfo($iSurveyId); $aData['surveyid'] = $iSurveyId; $aTokenTableFields = getTokenFieldsAndNames($iSurveyId); unset($aTokenTableFields['sent']); unset($aTokenTableFields['remindersent']); unset($aTokenTableFields['remindercount']); unset($aTokenTableFields['usesleft']); foreach ($aTokenTableFields as $sKey => $sValue) { if ($sValue['description'] != $sKey) { $sValue['description'] .= ' - ' . $sKey; } $aNewTokenTableFields[$sKey] = $sValue['description']; } $aData['aTokenTableFields'] = $aNewTokenTableFields; $this->_renderWrappedTemplate('token', array('tokenbar', 'csvupload'), $aData); } }
<?php $clang->eT("Choose the file to upload:"); ?> </label> <input type="file" name="the_file" required> </li> <li> <label for="characterset" id="characterset"> <?php $clang->eT("Character set of file:"); ?> </label> <select name="characterset"> <option value="auto" selected="selected">Automatic</option> <?php $encodingsarray = aEncodingsArray(); $encodingsarray_keys = array_keys($encodingsarray); $i = 0; foreach ($encodingsarray as $encoding) { ?> <option value="<?php echo $encodingsarray_keys[$i++]; ?> "><?php echo $encoding; ?> </option> <?php } ?> </select>
/** * import from csv */ public function import($iSurveyId) { $aData = array(); $iSurveyId = (int) $iSurveyId; if (!Permission::model()->hasSurveyPermission($iSurveyId, 'tokens', 'import')) { Yii::app()->session['flashmessage'] = gT("You do not have permission to access this page."); $this->getController()->redirect(array("/admin/survey/sa/view/surveyid/{$iSurveyId}")); } // CHECK TO SEE IF A TOKEN TABLE EXISTS FOR THIS SURVEY $bTokenExists = tableExists('{{tokens_' . $iSurveyId . '}}'); if (!$bTokenExists) { self::_newtokentable($iSurveyId); } $surveyinfo = Survey::model()->findByPk($iSurveyId)->surveyinfo; $aData['sidemenu']['state'] = false; $aData["surveyinfo"] = $surveyinfo; $aData['title_bar']['title'] = $surveyinfo['surveyls_title'] . "(" . gT("ID") . ":" . $iSurveyId . ")"; $aData['sidemenu']["token_menu"] = TRUE; $aData['token_bar']['closebutton']['url'] = 'admin/tokens/sa/index/surveyid/' . $iSurveyId; $this->registerScriptFile('ADMIN_SCRIPT_PATH', 'tokensimport.js'); $aEncodings = aEncodingsArray(); if (Yii::app()->request->isPostRequest) { $sUploadCharset = Yii::app()->request->getPost('csvcharset'); if (!array_key_exists($sUploadCharset, $aEncodings)) { $sUploadCharset = 'auto'; } $bFilterDuplicateToken = Yii::app()->request->getPost('filterduplicatetoken'); $bFilterBlankEmail = Yii::app()->request->getPost('filterblankemail'); $bAllowInvalidEmail = Yii::app()->request->getPost('allowinvalidemail'); $aAttrFieldNames = getAttributeFieldNames($iSurveyId); $aDuplicateList = array(); $aInvalidTokenList = array(); $aInvalidEmailList = array(); $aInvalidFormatList = array(); $aModelErrorList = array(); $aFirstLine = array(); $oFile = CUploadedFile::getInstanceByName("the_file"); $sPath = Yii::app()->getConfig('tempdir'); $sFileName = $sPath . '/' . randomChars(20); if ($_FILES['the_file']['error'] == 1 || $_FILES['the_file']['error'] == 2) { Yii::app()->setFlashMessage(sprintf(gT("Sorry, this file is too large. Only files up to %01.2f MB are allowed."), getMaximumFileUploadSize() / 1024 / 1024), 'error'); } elseif (strtolower($oFile->getExtensionName()) != 'csv') { Yii::app()->setFlashMessage(gT("Only CSV files are allowed."), 'error'); } elseif (!@$oFile->saveAs($sFileName)) { Yii::app()->setFlashMessage(sprintf(gT("Upload file not found. Check your permissions and path (%s) for the upload directory"), $sPath), 'error'); } else { $iRecordImported = 0; $iRecordCount = 0; $iRecordOk = 0; $iInvalidEmailCount = 0; // Count invalid email imported // This allows to read file with MAC line endings too @ini_set('auto_detect_line_endings', true); // open it and trim the ednings $aTokenListArray = file($sFileName); $sBaseLanguage = Survey::model()->findByPk($iSurveyId)->language; if (!Yii::app()->request->getPost('filterduplicatefields') || Yii::app()->request->getPost('filterduplicatefields') && count(Yii::app()->request->getPost('filterduplicatefields')) == 0) { $aFilterDuplicateFields = array('firstname', 'lastname', 'email'); } else { $aFilterDuplicateFields = Yii::app()->request->getPost('filterduplicatefields'); } $sSeparator = Yii::app()->request->getPost('separator'); $aMissingAttrFieldName = $aInvalideAttrFieldName = array(); foreach ($aTokenListArray as $buffer) { $buffer = @mb_convert_encoding($buffer, "UTF-8", $sUploadCharset); if ($iRecordCount == 0) { // Parse first line (header) from CSV $buffer = removeBOM($buffer); // We alow all field except tid because this one is really not needed. $aAllowedFieldNames = Token::model($iSurveyId)->tableSchema->getColumnNames(); if (($kTid = array_search('tid', $aAllowedFieldNames)) !== false) { unset($aAllowedFieldNames[$kTid]); } // Some header don't have same column name $aReplacedFields = array('invited' => 'sent', 'reminded' => 'remindersent'); switch ($sSeparator) { case 'comma': $sSeparator = ','; break; case 'semicolon': $sSeparator = ';'; break; default: $comma = substr_count($buffer, ','); $semicolon = substr_count($buffer, ';'); if ($semicolon > $comma) { $sSeparator = ';'; } else { $sSeparator = ','; } } $aFirstLine = str_getcsv($buffer, $sSeparator, '"'); $aFirstLine = array_map('trim', $aFirstLine); $aIgnoredColumns = array(); // Now check the first line for invalid fields foreach ($aFirstLine as $index => $sFieldname) { $aFirstLine[$index] = preg_replace("/(.*) <[^,]*>\$/", "\$1", $sFieldname); $sFieldname = $aFirstLine[$index]; if (!in_array($sFieldname, $aAllowedFieldNames)) { $aIgnoredColumns[] = $sFieldname; } if (array_key_exists($sFieldname, $aReplacedFields)) { $aFirstLine[$index] = $aReplacedFields[$sFieldname]; } // Attribute not in list if (strpos($aFirstLine[$index], 'attribute_') !== false and !in_array($aFirstLine[$index], $aAttrFieldNames) and Yii::app()->request->getPost('showwarningtoken')) { $aInvalideAttrFieldName[] = $aFirstLine[$index]; } } //compare attributes with source csv if (Yii::app()->request->getPost('showwarningtoken')) { $aMissingAttrFieldName = array_diff($aAttrFieldNames, $aFirstLine); // get list of mandatory attributes $allAttrFieldNames = GetParticipantAttributes($iSurveyId); //if it isn't mandantory field we don't need to show in warning if (!empty($aAttrFieldNames)) { if (!empty($aMissingAttrFieldName)) { foreach ($aMissingAttrFieldName as $index => $AttrFieldName) { if (isset($allAttrFieldNames[$AttrFieldName]) and strtolower($allAttrFieldNames[$AttrFieldName]["mandatory"]) != "y") { unset($aMissingAttrFieldName[$index]); } } } if (isset($aInvalideAttrFieldName) and !empty($aInvalideAttrFieldName)) { foreach ($aInvalideAttrFieldName as $index => $AttrFieldName) { if (isset($allAttrFieldNames[$AttrFieldName]) and strtolower($allAttrFieldNames[$AttrFieldName]["mandatory"]) != "y") { unset($aInvalideAttrFieldName[$index]); } } } } } } else { $line = str_getcsv($buffer, $sSeparator, '"'); if (count($aFirstLine) != count($line)) { $aInvalidFormatList[] = sprintf(gT("Line %s"), $iRecordCount); $iRecordCount++; continue; } $aWriteArray = array_combine($aFirstLine, $line); //kick out ignored columns foreach ($aIgnoredColumns as $column) { unset($aWriteArray[$column]); } $bDuplicateFound = false; $bInvalidEmail = false; $bInvalidToken = false; $aWriteArray['email'] = isset($aWriteArray['email']) ? trim($aWriteArray['email']) : ""; $aWriteArray['firstname'] = isset($aWriteArray['firstname']) ? $aWriteArray['firstname'] : ""; $aWriteArray['lastname'] = isset($aWriteArray['lastname']) ? $aWriteArray['lastname'] : ""; $aWriteArray['language'] = isset($aWriteArray['language']) ? $aWriteArray['language'] : $sBaseLanguage; if ($bFilterDuplicateToken) { $aParams = array(); $oCriteria = new CDbCriteria(); $oCriteria->condition = ""; foreach ($aFilterDuplicateFields as $field) { if (isset($aWriteArray[$field])) { $oCriteria->addCondition("{$field} = :{$field}"); $aParams[":{$field}"] = $aWriteArray[$field]; } } if (!empty($aParams)) { $oCriteria->params = $aParams; } $dupresult = TokenDynamic::model($iSurveyId)->count($oCriteria); if ($dupresult > 0) { $bDuplicateFound = true; $aDuplicateList[] = sprintf(gT("Line %s : %s %s (%s)"), $iRecordCount, $aWriteArray['firstname'], $aWriteArray['lastname'], $aWriteArray['email']); } } //treat blank emails if (!$bDuplicateFound && $bFilterBlankEmail && $aWriteArray['email'] == '') { $bInvalidEmail = true; $aInvalidEmailList[] = sprintf(gT("Line %s : %s %s"), $iRecordCount, CHtml::encode($aWriteArray['firstname']), CHtml::encode($aWriteArray['lastname'])); } if (!$bDuplicateFound && $aWriteArray['email'] != '') { $aEmailAddresses = preg_split("/(,|;)/", $aWriteArray['email']); foreach ($aEmailAddresses as $sEmailaddress) { if (!validateEmailAddress($sEmailaddress)) { if ($bAllowInvalidEmail) { $iInvalidEmailCount++; if (empty($aWriteArray['emailstatus']) || strtoupper($aWriteArray['emailstatus'] == "OK")) { $aWriteArray['emailstatus'] = "invalid"; } } else { $bInvalidEmail = true; $aInvalidEmailList[] = sprintf(gT("Line %s : %s %s (%s)"), $iRecordCount, CHtml::encode($aWriteArray['firstname']), CHtml::encode($aWriteArray['lastname']), CHtml::encode($aWriteArray['email'])); } } } } if (!$bDuplicateFound && !$bInvalidEmail && isset($aWriteArray['token']) && trim($aWriteArray['token']) != '') { if (trim($aWriteArray['token']) != sanitize_token($aWriteArray['token'])) { $aInvalidTokenList[] = sprintf(gT("Line %s : %s %s (%s) - token : %s"), $iRecordCount, CHtml::encode($aWriteArray['firstname']), CHtml::encode($aWriteArray['lastname']), CHtml::encode($aWriteArray['email']), CHtml::encode($aWriteArray['token'])); $bInvalidToken = true; } // We allways search for duplicate token (it's in model. Allow to reset or update token ? if (Token::model($iSurveyId)->count("token=:token", array(":token" => $aWriteArray['token']))) { $bDuplicateFound = true; $aDuplicateList[] = sprintf(gT("Line %s : %s %s (%s) - token : %s"), $iRecordCount, CHtml::encode($aWriteArray['firstname']), CHtml::encode($aWriteArray['lastname']), CHtml::encode($aWriteArray['email']), CHtml::encode($aWriteArray['token'])); } } if (!$bDuplicateFound && !$bInvalidEmail && !$bInvalidToken) { // unset all empty value foreach ($aWriteArray as $key => $value) { if ($aWriteArray[$key] == "") { unset($aWriteArray[$key]); } if (substr($value, 0, 1) == '"' && substr($value, -1) == '"') { // Fix CSV quote $value = substr($value, 1, -1); } } // Some default value : to be moved to Token model rules in future release ? // But think we have to accept invalid email etc ... then use specific scenario $oToken = Token::create($iSurveyId); if ($bAllowInvalidEmail) { $oToken->scenario = 'allowinvalidemail'; } foreach ($aWriteArray as $key => $value) { $oToken->{$key} = $value; } if (!$oToken->save()) { $errors = $oToken->getErrors(); $aModelErrorList[] = sprintf(gT("Line %s : %s"), $iRecordCount, print_r($errors, true)); } else { $iRecordImported++; } } $iRecordOk++; } $iRecordCount++; } $iRecordCount = $iRecordCount - 1; unlink($sFileName); $aData['aTokenListArray'] = $aTokenListArray; // Big array in memory, just for success ? $aData['iRecordImported'] = $iRecordImported; $aData['iRecordOk'] = $iRecordOk; $aData['iRecordCount'] = $iRecordCount; $aData['aFirstLine'] = $aFirstLine; // Seem not needed $aData['aDuplicateList'] = $aDuplicateList; $aData['aInvalidTokenList'] = $aInvalidTokenList; $aData['aInvalidFormatList'] = $aInvalidFormatList; $aData['aInvalidEmailList'] = $aInvalidEmailList; $aData['aModelErrorList'] = $aModelErrorList; $aData['iInvalidEmailCount'] = $iInvalidEmailCount; $aData['thissurvey'] = getSurveyInfo($iSurveyId); $aData['iSurveyId'] = $aData['surveyid'] = $iSurveyId; $aData['aInvalideAttrFieldName'] = $aInvalideAttrFieldName; $aData['aMissingAttrFieldName'] = $aMissingAttrFieldName; $this->_renderWrappedTemplate('token', array('csvimportresult'), $aData); Yii::app()->end(); } } // If there are error with file : show the form $aData['aEncodings'] = $aEncodings; asort($aData['aEncodings']); $aData['iSurveyId'] = $iSurveyId; $aData['thissurvey'] = getSurveyInfo($iSurveyId); $aData['surveyid'] = $iSurveyId; $aTokenTableFields = getTokenFieldsAndNames($iSurveyId); unset($aTokenTableFields['sent']); unset($aTokenTableFields['remindersent']); unset($aTokenTableFields['remindercount']); unset($aTokenTableFields['usesleft']); foreach ($aTokenTableFields as $sKey => $sValue) { if ($sValue['description'] != $sKey) { $sValue['description'] .= ' - ' . $sKey; } $aNewTokenTableFields[$sKey] = $sValue['description']; } $aData['aTokenTableFields'] = $aNewTokenTableFields; // Get default character set from global settings $thischaracterset = getGlobalSetting('characterset'); // If no encoding was set yet, use the old "auto" default if ($thischaracterset == "") { $thischaracterset = "auto"; } $aData['thischaracterset'] = $thischaracterset; $this->_renderWrappedTemplate('token', array('csvupload'), $aData); }
private function _displaySettings() { Yii::app()->loadHelper('surveytranslator'); // Save refurl from where global settings screen is called! $refurl = Yii::app()->getRequest()->getUrlReferrer(Yii::app()->createUrl('admin'), array('globalsettings')); // Some URLs are not to be allowed to refered back to. // These exceptions can be added to the $aReplacements array $aReplacements = array('admin/update/sa/step4b' => 'admin/sa/index', 'admin/user/sa/adduser' => 'admin/user/sa/index', 'admin/user/sa/setusertemplates' => 'admin/user/sa/index'); $data['title'] = "hi"; $data['message'] = "message"; foreach ($this->_checkSettings() as $key => $row) { $data[$key] = $row; } Yii::app()->loadLibrary('Date_Time_Converter'); $dateformatdetails = getDateFormatData(Yii::app()->session['dateformat']); $datetimeobj = new date_time_converter(dateShift(getGlobalSetting("updatelastcheck"), 'Y-m-d H:i:s', getGlobalSetting('timeadjust')), 'Y-m-d H:i:s'); $data['updatelastcheck'] = $datetimeobj->convert($dateformatdetails['phpdate'] . " H:i:s"); $data['updateavailable'] = getGlobalSetting("updateavailable") && Yii::app()->getConfig("updatable"); $data['updatable'] = Yii::app()->getConfig("updatable"); $data['updateinfo'] = getGlobalSetting("updateinfo"); $data['updatebuild'] = getGlobalSetting("updatebuild"); $data['updateversion'] = getGlobalSetting("updateversion"); $data['aUpdateVersions'] = json_decode(getGlobalSetting("updateversions"), true); $data['allLanguages'] = getLanguageData(false, Yii::app()->session['adminlang']); if (trim(Yii::app()->getConfig('restrictToLanguages')) == '') { $data['restrictToLanguages'] = array_keys($data['allLanguages']); $data['excludedLanguages'] = array(); } else { $data['restrictToLanguages'] = explode(' ', trim(Yii::app()->getConfig('restrictToLanguages'))); $data['excludedLanguages'] = array_diff(array_keys($data['allLanguages']), $data['restrictToLanguages']); } $data['fullpagebar']['savebutton']['form'] = 'frmglobalsettings'; $data['fullpagebar']['saveandclosebutton']['form'] = 'frmglobalsettings'; $data['fullpagebar']['closebutton']['url'] = 'admin/'; // Close button // List of available encodings $data['aEncodings'] = aEncodingsArray(); // Get current setting from DB $data['thischaracterset'] = getGlobalSetting('characterset'); $data['sideMenuBehaviour'] = getGlobalSetting('sideMenuBehaviour'); $data['aListOfThemeObjects'] = AdminTheme::getAdminThemeList(); $this->_renderWrappedTemplate('', 'globalSettings_view', $data); }
/** * import from csv */ function import($iSurveyId) { $iSurveyId = (int) $iSurveyId; if (!Permission::model()->hasSurveyPermission($iSurveyId, 'tokens', 'import')) { Yii::app()->session['flashmessage'] = gT("You do not have sufficient rights to access this page."); $this->getController()->redirect(array("/admin/survey/sa/view/surveyid/{$iSurveyId}")); } // CHECK TO SEE IF A TOKEN TABLE EXISTS FOR THIS SURVEY $bTokenExists = tableExists('{{tokens_' . $iSurveyId . '}}'); if (!$bTokenExists) { self::_newtokentable($iSurveyId); } App()->getClientScript()->registerScriptFile(Yii::app()->getConfig('adminscripts') . 'tokensimport.js'); $aEncodings = aEncodingsArray(); if (Yii::app()->request->isPostRequest) { $sUploadCharset = Yii::app()->request->getPost('csvcharset'); if (!array_key_exists($sUploadCharset, $aEncodings)) { $sUploadCharset = 'auto'; } $bFilterDuplicateToken = Yii::app()->request->getPost('filterduplicatetoken'); $bFilterBlankEmail = Yii::app()->request->getPost('filterblankemail'); $bAllowInvalidEmail = Yii::app()->request->getPost('allowinvalidemail'); $aAttrFieldNames = getAttributeFieldNames($iSurveyId); $aDuplicateList = array(); $aInvalidEmailList = array(); $aInvalidFormatList = array(); $aModelErrorList = array(); $aFirstLine = array(); $oFile = CUploadedFile::getInstanceByName("the_file"); $sPath = Yii::app()->getConfig('tempdir'); $sFileName = $sPath . '/' . randomChars(20); //$sFileTmpName=$oFile->getTempName(); /* More way to validate CSV ? $aCsvMimetypes = array( 'text/csv', 'text/plain', 'application/csv', 'text/comma-separated-values', 'application/excel', 'application/vnd.ms-excel', 'application/vnd.msexcel', 'text/anytext', 'application/octet-stream', 'application/txt', ); */ if (strtolower($oFile->getExtensionName()) != 'csv') { Yii::app()->setFlashMessage(gT("Only CSV files are allowed."), 'error'); } elseif (!@$oFile->saveAs($sFileName)) { Yii::app()->setFlashMessage(sprintf(gT("Upload file not found. Check your permissions and path (%s) for the upload directory"), $sPath), 'error'); } else { $iRecordImported = 0; $iRecordCount = 0; $iRecordOk = 0; $iInvalidEmailCount = 0; // Count invalid email imported // This allows to read file with MAC line endings too @ini_set('auto_detect_line_endings', true); // open it and trim the ednings $aTokenListArray = file($sFileName); $sBaseLanguage = Survey::model()->findByPk($iSurveyId)->language; if (!Yii::app()->request->getPost('filterduplicatefields') || Yii::app()->request->getPost('filterduplicatefields') && count(Yii::app()->request->getPost('filterduplicatefields')) == 0) { $aFilterDuplicateFields = array('firstname', 'lastname', 'email'); } else { $aFilterDuplicateFields = Yii::app()->request->getPost('filterduplicatefields'); } $sSeparator = Yii::app()->request->getPost('separator'); foreach ($aTokenListArray as $buffer) { $buffer = @mb_convert_encoding($buffer, "UTF-8", $sUploadCharset); if ($iRecordCount == 0) { // Parse first line (header) from CSV $buffer = removeBOM($buffer); // We alow all field except tid because this one is really not needed. $aAllowedFieldNames = Token::model($iSurveyId)->tableSchema->getColumnNames(); if (($kTid = array_search('tid', $aAllowedFieldNames)) !== false) { unset($aAllowedFieldNames[$kTid]); } // Some header don't have same column name $aReplacedFields = array('invited' => 'sent', 'reminded' => 'remindersent'); switch ($sSeparator) { case 'comma': $sSeparator = ','; break; case 'semicolon': $sSeparator = ';'; break; default: $comma = substr_count($buffer, ','); $semicolon = substr_count($buffer, ';'); if ($semicolon > $comma) { $sSeparator = ';'; } else { $sSeparator = ','; } } $aFirstLine = str_getcsv($buffer, $sSeparator, '"'); $aFirstLine = array_map('trim', $aFirstLine); $aIgnoredColumns = array(); // Now check the first line for invalid fields foreach ($aFirstLine as $index => $sFieldname) { $aFirstLine[$index] = preg_replace("/(.*) <[^,]*>\$/", "\$1", $sFieldname); $sFieldname = $aFirstLine[$index]; if (!in_array($sFieldname, $aAllowedFieldNames)) { $aIgnoredColumns[] = $sFieldname; } if (array_key_exists($sFieldname, $aReplacedFields)) { $aFirstLine[$index] = $aReplacedFields[$sFieldname]; } } } else { $line = str_getcsv($buffer, $sSeparator, '"'); if (count($aFirstLine) != count($line)) { $aInvalidFormatList[] = sprintf(gt("Line %s"), $iRecordCount); $iRecordCount++; continue; } $aWriteArray = array_combine($aFirstLine, $line); //kick out ignored columns foreach ($aIgnoredColumns as $column) { unset($aWriteArray[$column]); } $bDuplicateFound = false; $bInvalidEmail = false; $aWriteArray['email'] = isset($aWriteArray['email']) ? trim($aWriteArray['email']) : ""; $aWriteArray['firstname'] = isset($aWriteArray['firstname']) ? $aWriteArray['firstname'] : ""; $aWriteArray['lastname'] = isset($aWriteArray['lastname']) ? $aWriteArray['lastname'] : ""; $aWriteArray['language'] = isset($aWriteArray['language']) ? $aWriteArray['language'] : $sBaseLanguage; if ($bFilterDuplicateToken) { $aParams = array(); $oCriteria = new CDbCriteria(); $oCriteria->condition = ""; foreach ($aFilterDuplicateFields as $field) { if (isset($aWriteArray[$field])) { $oCriteria->addCondition("{$field} = :{$field}"); $aParams[":{$field}"] = $aWriteArray[$field]; } } if (!empty($aParams)) { $oCriteria->params = $aParams; } $dupresult = TokenDynamic::model($iSurveyId)->count($oCriteria); if ($dupresult > 0) { $bDuplicateFound = true; $aDuplicateList[] = sprintf(gt("Line %s : %s %s (%s)"), $iRecordCount, $aWriteArray['firstname'], $aWriteArray['lastname'], $aWriteArray['email']); } } //treat blank emails if (!$bDuplicateFound && $bFilterBlankEmail && $aWriteArray['email'] == '') { $bInvalidEmail = true; $aInvalidEmailList[] = sprintf(gt("Line %s : %s %s"), $iRecordCount, CHtml::encode($aWriteArray['firstname']), CHtml::encode($aWriteArray['lastname'])); } if (!$bDuplicateFound && $aWriteArray['email'] != '') { $aEmailAddresses = explode(';', $aWriteArray['email']); foreach ($aEmailAddresses as $sEmailaddress) { if (!validateEmailAddress($sEmailaddress)) { if ($bAllowInvalidEmail) { $iInvalidEmailCount++; if (empty($aWriteArray['emailstatus']) || strtoupper($aWriteArray['emailstatus'] == "OK")) { $aWriteArray['emailstatus'] = "invalid"; } } else { $bInvalidEmail = true; $aInvalidEmailList[] = sprintf(gt("Line %s : %s %s (%s)"), $iRecordCount, CHtml::encode($aWriteArray['firstname']), CHtml::encode($aWriteArray['lastname']), CHtml::encode($aWriteArray['email'])); } } } } if (!$bDuplicateFound && !$bInvalidEmail && isset($aWriteArray['token'])) { $aWriteArray['token'] = sanitize_token($aWriteArray['token']); // We allways search for duplicate token (it's in model. Allow to reset or update token ? if (Token::model($iSurveyId)->count("token=:token", array(":token" => $aWriteArray['token']))) { $bDuplicateFound = true; $aDuplicateList[] = sprintf(gt("Line %s : %s %s (%s) - token : %s"), $iRecordCount, CHtml::encode($aWriteArray['firstname']), CHtml::encode($aWriteArray['lastname']), CHtml::encode($aWriteArray['email']), CHtml::encode($aWriteArray['token'])); } } if (!$bDuplicateFound && !$bInvalidEmail) { // unset all empty value foreach ($aWriteArray as $key => $value) { if ($aWriteArray[$key] == "") { unset($aWriteArray[$key]); } if (substr($value, 0, 1) == '"' && substr($value, -1) == '"') { // Fix CSV quote $value = substr($value, 1, -1); } } // Some default value : to be moved to Token model rules in future release ? // But think we have to accept invalid email etc ... then use specific scenario $oToken = Token::create($iSurveyId); if ($bAllowInvalidEmail) { $oToken->scenario = 'allowinvalidemail'; } foreach ($aWriteArray as $key => $value) { $oToken->{$key} = $value; } if (!$oToken->save()) { tracevar($oToken->getErrors()); $aModelErrorList[] = sprintf(gt("Line %s : %s"), $iRecordCount, Chtml::errorSummary($oToken)); } else { $iRecordImported++; } } $iRecordOk++; } $iRecordCount++; } $iRecordCount = $iRecordCount - 1; unlink($sFileName); $aData['aTokenListArray'] = $aTokenListArray; // Big array in memory, just for success ? $aData['iRecordImported'] = $iRecordImported; $aData['iRecordOk'] = $iRecordOk; $aData['iRecordCount'] = $iRecordCount; $aData['aFirstLine'] = $aFirstLine; // Seem not needed $aData['aDuplicateList'] = $aDuplicateList; $aData['aInvalidFormatList'] = $aInvalidFormatList; $aData['aInvalidEmailList'] = $aInvalidEmailList; $aData['aModelErrorList'] = $aModelErrorList; $aData['iInvalidEmailCount'] = $iInvalidEmailCount; $aData['thissurvey'] = getSurveyInfo($iSurveyId); $aData['iSurveyId'] = $aData['surveyid'] = $iSurveyId; $this->_renderWrappedTemplate('token', array('tokenbar', 'csvpost'), $aData); Yii::app()->end(); } } // If there are error with file : show the form $aData['aEncodings'] = $aEncodings; $aData['iSurveyId'] = $iSurveyId; $aData['thissurvey'] = getSurveyInfo($iSurveyId); $aData['surveyid'] = $iSurveyId; $aTokenTableFields = getTokenFieldsAndNames($iSurveyId); unset($aTokenTableFields['sent']); unset($aTokenTableFields['remindersent']); unset($aTokenTableFields['remindercount']); unset($aTokenTableFields['usesleft']); foreach ($aTokenTableFields as $sKey => $sValue) { if ($sValue['description'] != $sKey) { $sValue['description'] .= ' - ' . $sKey; } $aNewTokenTableFields[$sKey] = $sValue['description']; } $aData['aTokenTableFields'] = $aNewTokenTableFields; $this->_renderWrappedTemplate('token', array('tokenbar', 'csvupload'), $aData); }
<?php eT("Choose the file to upload:"); ?> </label> <input type="file" name="the_file" required> </li> <li> <label for="characterset" id="characterset"> <?php eT("Character set of file:"); ?> </label> <select name="characterset"> <option value="auto" selected="selected">Automatic</option> <?php foreach (aEncodingsArray() as $key => $encoding) { ?> <option value="<?php echo $key; ?> "><?php echo $encoding; ?> </option> <?php } ?> </select> </li> <li> <label for="separatorused" id="separatorused">