/** * Create Batch and Data for do_stats * @author Sargis Davtyan */ public function createBatchContent_post() { $batchName = $this->input->post('batchName'); $fileName = $this->input->post('fileName'); $auto_start = $this->input->post('auto_start'); $customer = $this->input->post('customer'); $prefix = $this->input->post('prefix'); if ($fileName) { $this->load->helper('csv_helper'); $fileName = FCPATH . 'webroot/temp/' . $fileName . '.csv'; $lines = csv_to_array_all($fileName, detectDelimiter($fileName)); $columns = array(); if (count($lines) > 2) { foreach ($lines[0] as $key => $val) { switch (strtolower(trim($val))) { case 'site': $columns['site'] = $key; break; case 'tool id': $columns['tool_id'] = $key; break; case 'upc': $columns['upc'] = $key; break; case 'product name': $columns['product_name'] = $key; break; case 'short description': $columns['short_description'] = $key; break; case 'long description': $columns['long_description'] = $key; break; case 'url': $columns['url'] = $key; break; } } if (!isset($columns['url']) && isset($columns['tool_id'])) { if ($prefix) { $url_type = 1; } else { $url_type = 0; } } elseif (isset($columns['url'])) { $url_type = 2; } else { $url_type = 3; } $errors = 0; $duplicates = 0; $added = 0; if ($url_type !== 3) { if ($url_type !== 0) { $this->load->model('batches_model'); $batch_id = $this->batches_model->getIdByName($batchName); if (!$batch_id) { $this->load->model('customers_model'); $customer_id = $this->customers_model->getIdByName($customer); if ($customer_id) { $this->load->model('research_data_model'); $this->load->model('research_data_to_crawler_list_model'); $this->load->model('crawler_list_model'); $this->load->library('import_crawled_data'); $this->load->model('batches_combinations'); $data = array('type' => 1, 'creation_type' => null, 'selected_filters' => null, 'parent' => 0); $batch_id = $this->batches_model->insert($batchName, $customer_id, $data); if (intval($batch_id) > 0) { $combinations = $this->batches_combinations->generateCombinationsForBatch($batch_id); $maxId = $this->crawler_list_model->getMaxImportedDataId(); $ins = array(); $page_data = array(); foreach ($lines as $key => $row) { if ($key == 0) { continue; } if ($url_type === 1) { $url = $prefix . $row[$columns['tool_id']]; } else { $url = $row[$columns['url']]; } if ($url && ereg("(http(s)?\\:\\/\\/)?[a-zA-Z0-9\\-\\.]+\\.[a-zA-Z]{2,3}(\\/\\S*)?", $url)) { $res = $this->research_data_model->checkItemUrl($batch_id, $url); if (!$res) { $ins['category_id'] = 0; $ins['batch_id'] = $batch_id; $ins['url'] = $url; $research_data_id = $this->research_data_model->insert($ins); if ($research_data_id) { $crawler_list_id = $this->crawler_list_model->getByUrl($url); if (!$crawler_list_id) { $crawler_list_id = $this->crawler_list_model->insert($url, 0); } $this->research_data_to_crawler_list_model->insert($research_data_id, $crawler_list_id); $page_data['status'] = 'success'; $page_data['url'] = $url; $page_data['product_id'] = $crawler_list_id; if (isset($columns['product_name']) && $row[$columns['product_name']]) { $page_data['product_info']['product_name'] = $row[$columns['product_name']]; } else { $page_data['product_info']['product_name'] = ''; } if (isset($columns['upc']) && $row[$columns['upc']]) { $page_data['product_info']['upc'] = $row[$columns['upc']]; } else { $page_data['product_info']['upc'] = ''; } if (isset($columns['short_description']) && $row[$columns['short_description']]) { $page_data['product_info']['description'] = $row[$columns['short_description']]; } else { $page_data['product_info']['description'] = ''; } if (isset($columns['long_description']) && $row[$columns['long_description']]) { $page_data['product_info']['long_description'] = $row[$columns['long_description']]; } else { $page_data['product_info']['long_description'] = ''; } $page_data['page_attributes'] = array(); $page_data['sellers'] = array(); $page_data['classification'] = array(); $page_data['date'] = date('Y-m-d H:i:s'); $this->import_crawled_data->save($page_data); $added++; } } else { $duplicates++; } } else { $errors++; } } if (!empty($page_data)) { $this->load->model('settings_model'); $skey = 'batch_display_settings'; $batch_display_settings = $this->settings_model->get_system_value($skey); if ($batch_display_settings === false) { $this->settings_model->create_system($skey, array($batch_id)); } else { $batch_display_settings[] = $batch_id; $this->settings_model->update_system_value($skey, $batch_display_settings); } $user_id = $this->ion_auth->get_user_id(); $this->load->model('users_model'); $userData = $this->users_model->getUsersData($user_id); $this->load->library('email'); $config['protocol'] = 'sendmail'; $config['mailpath'] = '/usr/sbin/sendmail'; $config['charset'] = 'UTF-8'; $this->email->initialize($config); $this->email->from('*****@*****.**', 'Administrator'); $this->email->to('*****@*****.**'); $this->email->subject('New Batch and data created'); $messageHtml = 'A new batch has been added on ' . $_SERVER['HTTP_HOST']; $messageHtml .= '<br>Batch Name: ' . $batchName; $messageHtml .= '<br>User: '******'<br>TimeStamp: ' . date('Y-m-d H:i:s'); $this->email->message($messageHtml); $this->email->send(); if ($auto_start == "true") { $this->batches_model->updateBatchStatus($batch_id, 'do_stat'); } } unlink($fileName); $message = 'Errors: ' . $errors . '<br>Duplicates: ' . $duplicates . '<br>Added: ' . $added; $response = array('error' => false, 'message' => $message); } else { $response = array('error' => true, 'message' => 'Cannot create batch, please try again'); } } else { $response = array('error' => true, 'message' => 'Customer ' . $lines[1][$columns['site']] . ' not exist'); } } else { $response = array('error' => true, 'message' => 'Batch name already exist'); } } else { $response = array('error' => true, 'message' => 'Please add prefix', 'type' => 'prefix'); } } else { $response = array('error' => true, 'message' => 'Please add url or tool id in csv'); } } else { $response = array('error' => true, 'message' => 'CSV file is empty'); } } else { $response = array('error' => true, 'message' => 'CSV file not exist'); } $this->output->set_content_type('application/json')->set_output(json_encode($response)); }
/** * Upload data to `imported_data_parsed` table * @author Ruslan Ushakov **/ function match_urls_thread_new($choosen_file = '', $thread_max = 1, $mode = 'UploadUpdate', $updateAfterUpload = FALSE) { if ($mode !== 'UploadUpdate') { $mode = 'Update'; } if (!$choosen_file) { $choosen_file = $this->input->post('choosen_file'); } $this->load->model('settings_model'); $this->load->model('imported_data_parsed_model'); $this->load->model('temp_data_model'); $this->load->model('matchurls_data_model'); if ($mode == 'UploadUpdate') { $file = $this->config->item('csv_upload_dir') . $choosen_file; // $f_name = end(explode('/', $file)); } else { if ($choosen_file == 'all') { log_message('ERROR', 'This operation (all CSV update) not permitted'); $history_run['ERROR'] = 'This operation (all CSV update) not permitted'; return; } } // reopen DB in child process // $this->db->close(); // $this->db->initialize(); $this->temp_data_model->emptyTable('notfoundurls'); $this->temp_data_model->emptyTable('updated_items'); $this->matchurls_data_model->createCSVFileTables(); $this->temp_data_model->createMatchUrlsTable(); $this->temp_data_model->createNonFoundTable(); $this->temp_data_model->cUpdDataTable(); if ($mode == 'UploadUpdate' && $choosen_file != 'all') { // Mode Upload+Update $fileHandler = fopen($file, 'r'); if (!$fileHandler) { log_message('ERROR', 'File not open ' . $file); $history_run['ERROR'] = 'File not open ' . $file; $this->_save_history($history_run); return; } if ($fileCSV_id = $this->matchurls_data_model->getIdCSVFile($choosen_file)) { // $this->matchurls_data_model->deleteCSV($fileCSV_id); log_message('ERROR', 'DB cleared for file ' . $choosen_file . ' file_id=' . $fileCSV_id); } else { $fileCSV_id = $this->matchurls_data_model->addCSVFile($choosen_file); } } else { if ($choosen_file == 'all') { $fileCSV_ids = $this->matchurls_data_model->getIdsCSVFile(); if (!$fileCSV_ids) { log_message('ERROR', 'File not cacher in DB ' . $choosen_file); $history_run['ERROR'] = 'File not cacher in DB ' . $choosen_file; } } else { // Mode Update only $fileCSV_id = $this->matchurls_data_model->getIdCSVFile($choosen_file); if (!$fileCSV_id) { log_message('ERROR', 'File not cacher in DB ' . $choosen_file); $history_run['ERROR'] = 'File not cacher in DB ' . $choosen_file; } } } if ($mode == 'UploadUpdate' && $choosen_file != 'all') { $filenm = $this->config->item('csv_upload_dir') . $choosen_file; $this->load->helper('csv'); $lines = $this->csv_to_array($filenm, detectDelimiter($filenm)); if (isset($lines[0]) && count($lines[0]) > 2) { log_message('ERROR', 'Incorrect format of EOL in ' . $choosen_file); $history_run['ERROR'] = 'Incorrect format of EOL in ' . $choosen_file; $this->_save_history($history_run); } if (!empty($lines)) { $insert = $this->matchurls_data_model->addUrlsToMatch($fileCSV_id, $lines); //Add urls to table if (!$insert) { log_message('ERROR', 'File not cacher to DB ' . $choosen_file); $history_run['ERROR'] = 'File not cacher to DB ' . $choosen_file; } else { unset($lines); // unlink($filenm); } } } if (isset($fileCSV_ids) && is_array($fileCSV_ids) && !empty($fileCSV_ids)) { $this->settings_model->startUploadMatching($fileCSV_ids); } elseif (isset($fileCSV_id) && !empty($fileCSV_id)) { $this->settings_model->startUploadMatching($fileCSV_id); } if ($updateAfterUpload) { $this->settings_model->curl_async(base_url() . 'index.php/system/update_urls_threading_new/' . $choosen_file); } echo json_encode('true'); die; }