public function testCreateCsvEricsson() { $csvFileName = tempnam('/tmp', '_csvParserTest_'); file_put_contents($csvFileName, '8934071279000005005,GEM17223,2,SIM_Model,APN1,10.1.2.1,1,TM SPAIN'); $parser = new App_Parser_CsvParser($csvFileName); $list = $parser->parse(); unlink($csvFileName); try { $this->assertNotNull($this->stockMapper->createCsvEricsson($list)); } catch (Exception $e) { $this->fail($e->getMessage()); } }
public function getData($filename, $mime) { // I'm not sure... I don't like it. set_time_limit(0); /** * According to its mime type, call the * correct parser. */ switch ($mime) { case 'text/plain': /** * Try to parse the file as MSISDN data */ $parser = new \App_Parser_MSISDNParser($filename); $fileType = "csvMsisdn"; $data = $parser->parse(); $firstError = $parser->getErrors(0); if (!$data && $firstError && $firstError->code === ValidationCodes::FILE_TYPE_UNKNOWN) { // If it's not MSISDN data, try the CSV parser $parser = new \App_Parser_CsvParser($filename); $fileType = "csvData"; $data = $parser->parse(); } $firstError = $parser->getErrors(0); break; case 'application/xml': $parser = new \App_Parser_SimManufacturerParser($filename); $fileType = "xml"; $data = $parser->parse(); $firstError = $parser->getErrors(0); break; default: $firstError = \App_Parser_AbstractParser::createUnknownFileTypeError(); throw new AppEx\StockParserException('Invalid parameter value: attached file', array('errorMessages' => array($firstError))); } if ($firstError && $firstError->code === ValidationCodes::FILE_TYPE_UNKNOWN) { \App::log()->warn("Error parsing file: \n" . implode("\n", $parser->getErrors())); throw new AppEx\StockParserException('Invalid parameter value: attached file', array('errorMessages' => array($firstError))); } $errors = $parser->getErrors(); $errors = array_reverse($errors, false); $count = $parser->getItemParsedCount(); if (!$data) { \App::log()->warn("Error parsing file: \n" . implode("\n", $errors)); throw new AppEx\InvalidFileContentException('Invalid parameter value: attached file', array('errorMessages' => $errors, 'fileType' => $fileType, 'simParsed' => $count)); } return array('data' => $data, 'errors' => $errors, 'count' => $count); }
/** * Attempt to parse the file and return an array * with the results. */ public function parse() { $list = array(); $errors = array(); if ($this->checkFile()) { try { $fd = fopen($this->_fileName, 'r'); } catch (Exception $e) { // File isn't readable $this->_addError(self::createReadingFileError()); return false; } $sep = \App_Parser_CsvParser::getCsvSeparator($fd); $iter = 0; while (($line = fgetcsv($fd, 256, $sep)) !== false) { $line = array_map('trim', $line); if (count($line) <= 1 && empty($line[0])) { continue; } $iter++; $record = array(); if (count($line) !== 2) { $this->_addError(self::createUnknownFileTypeError()); \App::log()->info("MSISDN files must have 2 columns. It is not a MSISDN file."); fclose($fd); return false; } list($record['icc'], $record['msisdn']) = $line; if (($valid = $this->validateRecord($record)) === true) { $list[] = $record; } else { $valid->line = $iter; if (!empty($record['icc'])) { $valid->entity = 'icc'; $valid->entityId = $record['icc']; } $this->_addError($valid); } } fclose($fd); } else { // File isn't readable $this->_addError(self::createReadingFileError()); return false; } if (!empty($this->_errors)) { $errors = implode("\n", $this->_errors); \App::log()->warn("File some errors in CSV file: \n" . $errors); } $this->_itemParsedCount = $iter; return $this->identifyList($list, 'msisdn', 'preinventory_associate_msisdn'); }
/** * Attempt to parse the file and return an array * with the results. * * CSV row: * [0] icc * [1] imsi * [2] msisdn * [3] logistics * [4] hlr_id * [5] pin1 * [6] pin2 * [7] puk1 * [8] puk2 */ public function parse() { $list = array(); $errors = array(); if ($this->checkFile()) { try { $fd = fopen($this->_fileName, 'r'); } catch (Exception $e) { // File isn't readable $this->_addError(self::createReadingFileError()); return false; } $sep = \App_Parser_CsvParser::getCsvSeparator($fd); $iter = 0; while (($line = fgetcsv($fd, 1024, $sep)) !== false) { $n = count($line); if ($n <= 1 && empty($line[0])) { continue; } $iter++; $record = array(); if ($n < 5 || $n > 9) { $this->_addError(self::createUnknownFileTypeError()); \App::log()->info("Legacy files must have at between 5 and 9 columns. It is not a Legacy file."); fclose($fd); return false; } $line = array_map('trim', $line); $record = array('icc' => $line[0], 'imsi' => $line[1], 'msisdn' => $line[2], 'logistics' => $line[3], 'hlrId' => $line[4], 'pin1' => @$line[5], 'pin2' => @$line[6], 'puk1' => @$line[7], 'puk2' => @$line[8]); if (($valid = $this->validateRecord($record)) === true) { $list[] = $record; } else { $valid->line = $iter; $this->_addError($valid); } } fclose($fd); } else { $this->_addError(self::createReadingFileError()); return false; } if (!empty($this->_errors)) { $errors = implode("\n", $this->_errors); \App::log()->warn("File some errors in Legacy file: \n" . $errors); } $this->_itemParsedCount = $iter; return $this->identifyList($list, 'legacy', 'preinventory_legacy'); }
/** * * @param string $filename */ public function uploadBulk($filename) { // Open file if (!stat($filename)) { throw new \Application\Exceptions\UnexpectedException("Error opening file {$filename}"); } $fd = fopen($filename, 'r'); $sep = \App_Parser_CsvParser::getCsvSeparator($fd); // Request $sims = array(); $validator = new SimBulkValidate(); $validHeaders = $this->getUploadBulkHeaders(); $filter = \Zend_Controller_Action_HelperBroker::getStaticHelper('FilterNotAllowedFields'); // CSV $header = fgetcsv($fd, 0, $sep); // Header $prows = 0; // Packet rows $irows = 1; // Iteration rows $nrows = 1; // Total rows $ncols = count($header); // Total columns $validH = false; foreach ($header as $h) { if (isset($validHeaders[$h])) { $validH = true; break; } } // Check lines readed if (!$validH) { \App::log()->debug("[UpdateBulk] No valid headers"); throw new ValidateException("No valid headers", ValidationCodes::FILE_NO_VALID_HEADERS); } if (in_array('locationManual_latitude', $header) && !in_array('locationManual_longitude', $header) || in_array('locationManual_longitude', $header) && !in_array('locationManual_latitude', $header)) { \App::log()->debug("[UpdateBulk] No valid headers: location requires latitude and longitude"); throw new ValidateException("No valid headers: location requires latitude and longitude", ValidationCodes::FILE_NO_VALID_HEADERS_LOCATION); } $bulkCount = \App::config('ericssonserviceCallItemCount', self::BULK_UPDATE_DEFAULT); // I'm not sure... I don't like it. set_time_limit(0); $watcher = $this->getMapper()->createFileWatcher(); $txId = uniqid("bulk"); $watcher->entityIds = array($txId); $watcher->params->action = "bulkSimUpdate"; WatcherService::getInstance()->create($watcher); $errors = array(); $warnings = array(); $ntxs = 0; // Rows while (($line = fgetcsv($fd, 0, $sep)) !== false) { // Next line has been readed $nrows++; // Check columns if (count($line) !== $ncols) { $errors[] = new Model\ErrorModel(array('line' => $nrows, 'description' => "Incorrect format (number of columns)", 'level' => Model\ErrorModel::ERROR_LEVEL_ERROR, 'code' => ValidationCodes::FILE_READING_ERR)); continue; } // Create sim $data = array(); foreach ($header as $key => $name) { // \App::log()->debug("[UpdateBulk] $name : $key"); if (!isset($validHeaders[$name])) { // Ignore invalid columns // \App::log()->warn("[UpdateBulk] Ignoring $name column"); continue; } $value = $line[$key]; if (preg_match('/^=\\"(?P<value>.*)\\"$/', $value, $matches)) { $value = $matches['value']; } // GLOBALPORTAL-28668 // if (!empty($value)) { // \App::log()->debug("[UpdateBulk] $name : $value"); if (isset($value) && (!empty($value) || is_numeric($value))) { // \App::log()->debug("[UpdateBulk] TRUE $name : $value"); // Remove field? // See SimBaseMapper _mapModelToEricssonModel method, // SimValidate and App_Validate_ReferenceIndex to understand it if ($value === '-' && $name !== 'staticIpApnIndex') { $value = ''; } // Process field if (strpos($name, 'apn_') !== false) { // In order to remove the current value of a SIM´s field, // the character - must be indicated $index = (int) substr($name, strlen('apn_apn')) - 1; $data['apns'][$index] = $value; } else { if (strpos($name, 'locationManual_') !== false) { $value = str_replace(',', '.', $value); if (!is_numeric($value)) { $warnings[] = new Model\ErrorModel(array('line' => $nrows, 'column' => $name, 'description' => "Invalid value", 'level' => Model\ErrorModel::ERROR_LEVEL_WARN, 'code' => ValidationCodes::INVALID_VALUE)); } else { $subname = substr($name, strlen('locationManual_')); $value = floatval(str_replace(',', '.', $value)); $data['locationManual'][$subname] = (int) round($value * pow(10, 6)); } } else { if ($name == 'LTE_status') { if ($value != SimModel::LTE_ACTIVE && $value != SimModel::LTE_INACTIVE) { $warnings[] = new Model\ErrorModel(array('line' => $nrows, 'column' => $name, 'description' => "Invalid value", 'level' => Model\ErrorModel::ERROR_LEVEL_WARN, 'code' => ValidationCodes::INVALID_VALUE)); } else { $data['lteEnabled'] = $value == SimModel::LTE_ACTIVE ? true : false; } } else { $data[$name] = $value; } } } } } // Create and validate sim $sim = new SimModel($data); $v = $this->validate($sim, false, $validator); if ($v === true) { // Backup id $ids = $sim->getIds(); $type = key($ids); $value = current($ids); //Inject organization $org = \App::getOrgUserLogged(); switch ($org->getType()) { case OrgMasterModel::ORG_TYPE: $sim->setMaster($org); break; case OrgServiceProviderModel::ORG_TYPE: $sim->setServiceProviderCommercial($org); $sim->setServiceProviderEnabler($org); break; case OrgCustomerModel::ORG_TYPE: $sim->setCustomer($org); break; case OrgAggregatorModel::ORG_TYPE: $sim->setAggregator($org); break; case OrgEndUserModel::ORG_TYPE: $sim->setEndUser($org); break; } // Filter by permissions $filter->direct('update_field', $sim); // Recover id and add sim to request $sim->{$type} = $value; $sims[] = $sim; $prows++; } else { \App::log()->warn("[UpdateBulk] Ignoring invalid sim: " . json_encode($v)); // Sending first validation error ONLY? foreach ($validator->getValidationCodes() as $field => $code) { $errors[] = new Model\ErrorModel(array('line' => $nrows, 'description' => $field, 'level' => Model\ErrorModel::ERROR_LEVEL_WARN, 'code' => $code ?: ValidationCodes::MODEL_SIM)); } } // Wait until packet is full if ($prows == $bulkCount) { // Send to Ericsson $this->_uploadBulk($sims, $errors, $irows, $nrows, $watcher); $ntxs++; // Reset packet list $sims = array(); $prows = 0; // Update CSV line position $irows = $nrows + 1; } } // Ensure all sims have been sent (last packet) if (!empty($sims)) { // Send to Ericsson $this->_uploadBulk($sims, $errors, $irows, $nrows, $watcher); $ntxs++; // Reset packet list (memory propouses) $sims = array(); } // Check lines readed if ($nrows < 2) { \App::log()->debug("[UpdateBulk] Ignoring empty file"); $watcher->delete(); throw new ValidateException("Missing file rows"); } $event = $this->getMapper()->constructEventToTransaction(); $event->entityId = $txId; // Add error code suffix if (isset($errors) && is_array($errors)) { foreach ($errors as $errMess) { require_once APPLICATION_PATH . '/modules/default/controllers/ErrorController.php'; $errMess->code = \ErrorController::finishErrorCode($errMess->code); } } $eventData = array('simParsed' => $nrows - 1); if (!empty($errors) || !empty($warnings)) { $eventData['hasFailures'] = true; if (!empty($errors)) { $eventData['message']['failed'] = $errors; } if (!empty($warnings)) { $eventData['message']['warnings'] = $warnings; } } else { $eventData['hasFailures'] = false; } $event->eventData = $eventData; $compressor = new ErrorModelCompressEvent(); $compressor->compress($event); WatcherService::getInstance()->publishEvent($event); $nerr = count($errors); \App::audit("Bulk update ({$nrows} sims in {$ntxs} requests with {$nerr} errors)", null); return $watcher->reload(); }