/**
* Create Batch and Data for do_stats
* @author Sargis Davtyan
*/
public function createBatchContent_post()
{
$batchName = $this->input->post('batchName');
$fileName = $this->input->post('fileName');
$auto_start = $this->input->post('auto_start');
$customer = $this->input->post('customer');
$prefix = $this->input->post('prefix');
if ($fileName) {
$this->load->helper('csv_helper');
$fileName = FCPATH . 'webroot/temp/' . $fileName . '.csv';
$lines = csv_to_array_all($fileName, detectDelimiter($fileName));
$columns = array();
if (count($lines) > 2) {
foreach ($lines[0] as $key => $val) {
switch (strtolower(trim($val))) {
case 'site':
$columns['site'] = $key;
break;
case 'tool id':
$columns['tool_id'] = $key;
break;
case 'upc':
$columns['upc'] = $key;
break;
case 'product name':
$columns['product_name'] = $key;
break;
case 'short description':
$columns['short_description'] = $key;
break;
case 'long description':
$columns['long_description'] = $key;
break;
case 'url':
$columns['url'] = $key;
break;
}
}
if (!isset($columns['url']) && isset($columns['tool_id'])) {
if ($prefix) {
$url_type = 1;
} else {
$url_type = 0;
}
} elseif (isset($columns['url'])) {
$url_type = 2;
} else {
$url_type = 3;
}
$errors = 0;
$duplicates = 0;
$added = 0;
if ($url_type !== 3) {
if ($url_type !== 0) {
$this->load->model('batches_model');
$batch_id = $this->batches_model->getIdByName($batchName);
if (!$batch_id) {
$this->load->model('customers_model');
$customer_id = $this->customers_model->getIdByName($customer);
if ($customer_id) {
$this->load->model('research_data_model');
$this->load->model('research_data_to_crawler_list_model');
$this->load->model('crawler_list_model');
$this->load->library('import_crawled_data');
$this->load->model('batches_combinations');
$data = array('type' => 1, 'creation_type' => null, 'selected_filters' => null, 'parent' => 0);
$batch_id = $this->batches_model->insert($batchName, $customer_id, $data);
if (intval($batch_id) > 0) {
$combinations = $this->batches_combinations->generateCombinationsForBatch($batch_id);
$maxId = $this->crawler_list_model->getMaxImportedDataId();
$ins = array();
$page_data = array();
foreach ($lines as $key => $row) {
if ($key == 0) {
continue;
}
if ($url_type === 1) {
$url = $prefix . $row[$columns['tool_id']];
} else {
$url = $row[$columns['url']];
}
if ($url && ereg("(http(s)?\\:\\/\\/)?[a-zA-Z0-9\\-\\.]+\\.[a-zA-Z]{2,3}(\\/\\S*)?", $url)) {
$res = $this->research_data_model->checkItemUrl($batch_id, $url);
if (!$res) {
$ins['category_id'] = 0;
$ins['batch_id'] = $batch_id;
$ins['url'] = $url;
$research_data_id = $this->research_data_model->insert($ins);
if ($research_data_id) {
$crawler_list_id = $this->crawler_list_model->getByUrl($url);
if (!$crawler_list_id) {
$crawler_list_id = $this->crawler_list_model->insert($url, 0);
}
$this->research_data_to_crawler_list_model->insert($research_data_id, $crawler_list_id);
$page_data['status'] = 'success';
$page_data['url'] = $url;
$page_data['product_id'] = $crawler_list_id;
if (isset($columns['product_name']) && $row[$columns['product_name']]) {
$page_data['product_info']['product_name'] = $row[$columns['product_name']];
} else {
$page_data['product_info']['product_name'] = '';
}
if (isset($columns['upc']) && $row[$columns['upc']]) {
$page_data['product_info']['upc'] = $row[$columns['upc']];
} else {
$page_data['product_info']['upc'] = '';
}
if (isset($columns['short_description']) && $row[$columns['short_description']]) {
$page_data['product_info']['description'] = $row[$columns['short_description']];
} else {
$page_data['product_info']['description'] = '';
}
if (isset($columns['long_description']) && $row[$columns['long_description']]) {
$page_data['product_info']['long_description'] = $row[$columns['long_description']];
} else {
$page_data['product_info']['long_description'] = '';
}
$page_data['page_attributes'] = array();
$page_data['sellers'] = array();
$page_data['classification'] = array();
$page_data['date'] = date('Y-m-d H:i:s');
$this->import_crawled_data->save($page_data);
$added++;
}
} else {
$duplicates++;
}
} else {
$errors++;
}
}
if (!empty($page_data)) {
$this->load->model('settings_model');
$skey = 'batch_display_settings';
$batch_display_settings = $this->settings_model->get_system_value($skey);
if ($batch_display_settings === false) {
$this->settings_model->create_system($skey, array($batch_id));
} else {
$batch_display_settings[] = $batch_id;
$this->settings_model->update_system_value($skey, $batch_display_settings);
}
$user_id = $this->ion_auth->get_user_id();
$this->load->model('users_model');
$userData = $this->users_model->getUsersData($user_id);
$this->load->library('email');
$config['protocol'] = 'sendmail';
$config['mailpath'] = '/usr/sbin/sendmail';
$config['charset'] = 'UTF-8';
$this->email->initialize($config);
$this->email->from('*****@*****.**', 'Administrator');
$this->email->to('*****@*****.**');
$this->email->subject('New Batch and data created');
$messageHtml = 'A new batch has been added on ' . $_SERVER['HTTP_HOST'];
$messageHtml .= '<br>Batch Name: ' . $batchName;
$messageHtml .= '<br>User: '******'<br>TimeStamp: ' . date('Y-m-d H:i:s');
$this->email->message($messageHtml);
$this->email->send();
if ($auto_start == "true") {
$this->batches_model->updateBatchStatus($batch_id, 'do_stat');
}
}
unlink($fileName);
$message = 'Errors: ' . $errors . '<br>Duplicates: ' . $duplicates . '<br>Added: ' . $added;
$response = array('error' => false, 'message' => $message);
} else {
$response = array('error' => true, 'message' => 'Cannot create batch, please try again');
}
} else {
$response = array('error' => true, 'message' => 'Customer ' . $lines[1][$columns['site']] . ' not exist');
}
} else {
$response = array('error' => true, 'message' => 'Batch name already exist');
}
} else {
$response = array('error' => true, 'message' => 'Please add prefix', 'type' => 'prefix');
}
} else {
$response = array('error' => true, 'message' => 'Please add url or tool id in csv');
}
} else {
$response = array('error' => true, 'message' => 'CSV file is empty');
}
} else {
$response = array('error' => true, 'message' => 'CSV file not exist');
}
$this->output->set_content_type('application/json')->set_output(json_encode($response));
}
/**
* Upload data to `imported_data_parsed` table
* @author Ruslan Ushakov
**/
function match_urls_thread_new($choosen_file = '', $thread_max = 1, $mode = 'UploadUpdate', $updateAfterUpload = FALSE)
{
if ($mode !== 'UploadUpdate') {
$mode = 'Update';
}
if (!$choosen_file) {
$choosen_file = $this->input->post('choosen_file');
}
$this->load->model('settings_model');
$this->load->model('imported_data_parsed_model');
$this->load->model('temp_data_model');
$this->load->model('matchurls_data_model');
if ($mode == 'UploadUpdate') {
$file = $this->config->item('csv_upload_dir') . $choosen_file;
// $f_name = end(explode('/', $file));
} else {
if ($choosen_file == 'all') {
log_message('ERROR', 'This operation (all CSV update) not permitted');
$history_run['ERROR'] = 'This operation (all CSV update) not permitted';
return;
}
}
// reopen DB in child process
// $this->db->close();
// $this->db->initialize();
$this->temp_data_model->emptyTable('notfoundurls');
$this->temp_data_model->emptyTable('updated_items');
$this->matchurls_data_model->createCSVFileTables();
$this->temp_data_model->createMatchUrlsTable();
$this->temp_data_model->createNonFoundTable();
$this->temp_data_model->cUpdDataTable();
if ($mode == 'UploadUpdate' && $choosen_file != 'all') {
// Mode Upload+Update
$fileHandler = fopen($file, 'r');
if (!$fileHandler) {
log_message('ERROR', 'File not open ' . $file);
$history_run['ERROR'] = 'File not open ' . $file;
$this->_save_history($history_run);
return;
}
if ($fileCSV_id = $this->matchurls_data_model->getIdCSVFile($choosen_file)) {
// $this->matchurls_data_model->deleteCSV($fileCSV_id);
log_message('ERROR', 'DB cleared for file ' . $choosen_file . ' file_id=' . $fileCSV_id);
} else {
$fileCSV_id = $this->matchurls_data_model->addCSVFile($choosen_file);
}
} else {
if ($choosen_file == 'all') {
$fileCSV_ids = $this->matchurls_data_model->getIdsCSVFile();
if (!$fileCSV_ids) {
log_message('ERROR', 'File not cacher in DB ' . $choosen_file);
$history_run['ERROR'] = 'File not cacher in DB ' . $choosen_file;
}
} else {
// Mode Update only
$fileCSV_id = $this->matchurls_data_model->getIdCSVFile($choosen_file);
if (!$fileCSV_id) {
log_message('ERROR', 'File not cacher in DB ' . $choosen_file);
$history_run['ERROR'] = 'File not cacher in DB ' . $choosen_file;
}
}
}
if ($mode == 'UploadUpdate' && $choosen_file != 'all') {
$filenm = $this->config->item('csv_upload_dir') . $choosen_file;
$this->load->helper('csv');
$lines = $this->csv_to_array($filenm, detectDelimiter($filenm));
if (isset($lines[0]) && count($lines[0]) > 2) {
log_message('ERROR', 'Incorrect format of EOL in ' . $choosen_file);
$history_run['ERROR'] = 'Incorrect format of EOL in ' . $choosen_file;
$this->_save_history($history_run);
}
if (!empty($lines)) {
$insert = $this->matchurls_data_model->addUrlsToMatch($fileCSV_id, $lines);
//Add urls to table
if (!$insert) {
log_message('ERROR', 'File not cacher to DB ' . $choosen_file);
$history_run['ERROR'] = 'File not cacher to DB ' . $choosen_file;
} else {
unset($lines);
// unlink($filenm);
}
}
}
if (isset($fileCSV_ids) && is_array($fileCSV_ids) && !empty($fileCSV_ids)) {
$this->settings_model->startUploadMatching($fileCSV_ids);
} elseif (isset($fileCSV_id) && !empty($fileCSV_id)) {
$this->settings_model->startUploadMatching($fileCSV_id);
}
if ($updateAfterUpload) {
$this->settings_model->curl_async(base_url() . 'index.php/system/update_urls_threading_new/' . $choosen_file);
}
echo json_encode('true');
die;
}
