function import_assets_from_csv($filename, $iic, $ctx, $import_type)
{
//Process status
$summary = array('general' => array('status' => '', 'data' => '', 'statistics' => array('total' => 0, 'warnings' => 0, 'errors' => 0, 'saved' => 0)), 'by_hosts' => array());
$db = new ossim_db();
$conn = $db->connect();
$str_data = file_get_contents($filename);
if ($str_data === FALSE) {
$summary['general']['status'] = 'error';
$summary['general']['data']['errors'] = _('Failed to read data from CSV file');
$summary['general']['statistics']['errors'] = 1;
return $summary;
}
$array_data = preg_split('/\\n|\\r/', $str_data);
foreach ($array_data as $k => $v) {
if (trim($v) != '') {
$data[] = explode('";"', trim($v));
}
}
/*************************************************************************************************************************************
* From asset section:
* - Version 4.x.x or higher: "IP (IP1,IP2,...)";"Hostname";"FQDNs(FQDN1,FQDN2,...)";"Description";"Asset value";"Operating System";
* "Latitude";"Longitude";"Host ID";"External Asset";"Device Types(Type1,Type2,...)"
*
* - Version 3.x.x: "IP"*;"Hostname";"FQDNs(FQDN1,FQDN2,...)";"Description";"Asset value";"Sensors(Sensor1,Sensor2,...)";
* "Operating System";"Latitude";"Longitude"
*
* From welcome wizard:
* - Version 4.x.x or higher: "IP (IP1,IP2,...)";"Hostname";"Description";"Operating System";"Device Type(Type1,Type2,...)"
*
**************************************************************************************************************************************/
//Check file size
if (count($data) <= 0 || count($data) == 1 && preg_match('/IP/', $data[0][0])) {
$summary['general']['status'] = 'error';
$summary['general']['data'] = _('CSV file is empty');
$summary['general']['statistics']['errors'] = 1;
return $summary;
}
//Check importation type and headers
$csv_headers = array();
if ($import_type == 'hosts') {
if (preg_match('/Operating System/', $data[0][5]) || preg_match('/Sensors/', $data[0][5])) {
$csv_headers = array_shift($data);
} else {
$summary['general']['status'] = 'error';
$summary['general']['data'] = _('Headers not found');
$summary['general']['statistics']['errors'] = 1;
return $summary;
}
}
//Setting total hosts to import
$summary['general']['statistics']['total'] = count($data);
//Getting all Operating System
$all_os = Properties::get_all_os();
//Getting devices types
$all_devices = array();
$aux_all_devices = Devices::get_all_for_filter($conn);
$_all_devices = $aux_all_devices[0];
foreach ($_all_devices as $d_data) {
$d_key = $d_data['type_name'];
$d_key .= $d_data['subtype_id'] != 0 ? ':' . $d_data['subtype_name'] : '';
$all_devices[$d_key] = $d_data['type_id'] . ':' . $d_data['subtype_id'];
}
//Allowed sensors
$filters = array('where' => "acl_sensors.entity_id = UNHEX('{$ctx}')");
$a_sensors = Av_sensor::get_basic_list($conn, $filters);
$sensor_ids = array_keys($a_sensors);
if (count($sensor_ids) == 0) {
$summary['general']['status'] = 'error';
$s_error_msg = Session::is_pro() ? _('There is no sensors for this context') : _('There is no sensors for this IP address');
$summary['general']['data'] = $s_error_msg;
$summary['general']['statistics']['errors'] = 1;
return $summary;
}
Util::disable_perm_triggers($conn, TRUE);
foreach ($data as $k => $v) {
//Clean previous errors
ossim_clean_error();
$num_line = $k + 1;
//Set default status
$summary['by_hosts'][$num_line]['status'] = 'error';
//Check file format
$cnd_1 = $import_type == 'hosts' && count($v) < 9;
$cnd_2 = $import_type == 'welcome_wizard_hosts' && count($v) < 5;
if ($cnd_1 || $cnd_2) {
$summary['by_hosts'][$num_line]['errors']['Format'] = _('Number of fields is incorrect');
$summary['general']['statistics']['errors']++;
continue;
}
//Clean values
$param = array();
$index = 0;
$max_index = count($v) - 1;
foreach ($v as $field) {
$parameter = trim($field);
if ($index == 0) {
$pattern = '/^\\"|^\'/';
$param[] = preg_replace($pattern, '', $parameter);
} else {
if ($index == $max_index) {
$pattern = '/\\"$|\'$/';
$param[] = preg_replace($pattern, '', $parameter);
} else {
$param[] = $parameter;
}
}
$index++;
}
//Values
$is_in_db = FALSE;
$host_id = '';
$sensors = $sensor_ids;
$csv_ips = preg_replace("/\\s+/", '', $param[0]);
if (!empty($param[1])) {
$name = $param[1];
} else {
$aux_name = str_replace(' ', '', $csv_ips);
$aux_name = str_replace(',', '-', $aux_name);
$name = Asset_host::get_autodetected_name($aux_name);
}
if ($import_type == 'hosts') {
$fqdns = $param[2];
$descr = $param[3];
$asset_value = !empty($param[4]) ? $param[4] : 2;
if (preg_match('/Host ID/', $csv_headers[8])) {
$os = $param[5];
$latitude = floatval($param[6]);
$longitude = floatval($param[7]);
$external = empty($param[9]) ? 0 : intval($param[9]);
$csv_devices = $param[10];
} else {
$os = $param[6];
$latitude = floatval($param[7]);
$longitude = floatval($param[8]);
$external = 0;
$csv_devices = '';
}
} else {
$descr = $param[2];
$os = $param[3];
$latitude = 0;
$longitude = 0;
$asset_value = 2;
$external = 0;
$csv_devices = $param[4];
}
//Permissions
$can_i_create_assets = Session::can_i_create_assets();
$can_i_modify_ips = TRUE;
//IPs
if (!ossim_valid($csv_ips, OSS_IP_ADDR, 'illegal:' . _('IP'))) {
$summary['by_hosts'][$num_line]['errors']['IP'] = ossim_get_error_clean();
$summary['general']['statistics']['errors']++;
continue;
}
//Check Host ID: Is there a host registered in the System?
$host_ids = Asset_host::get_id_by_ips($conn, $csv_ips, $ctx);
$host_id = key($host_ids);
if (!empty($host_id)) {
$is_in_db = TRUE;
} else {
$host_id = Util::uuid();
}
// Special case: Forced Host ID [Version 4.x.x or higher]
if ($import_type == 'hosts' && preg_match('/Host ID/', $csv_headers[8]) && valid_hex32($param[8])) {
$csv_hosts_id = strtoupper($param[8]);
if ($is_in_db == TRUE && $csv_hosts_id != $host_id) {
$id_error_msg = _('Host is already registered in the System with another Host ID');
$summary['by_hosts'][$num_line]['errors']['Host'] = $id_error_msg;
$summary['general']['statistics']['errors']++;
continue;
} else {
if ($is_in_db == FALSE) {
$host_id = $csv_hosts_id;
// Save host ID to insert it
}
}
}
//Hostname
if (!empty($iic)) {
$name = clean_iic($name);
}
if (!ossim_valid($name, OSS_HOST_NAME, 'illegal:' . _('Hostname'))) {
ossim_clean_error();
$name = Asset_host::create_valid_name($name);
$warning_msg = _('Hostname does not match with RFC 1123 specifications') . '<br/>' . _('Hostname will be replaced by') . ": <strong>{$name}</strong>";
$summary['by_hosts'][$num_line]['warnings']['Hostname'] = $warning_msg;
$summary['by_hosts'][$num_line]['status'] = 'warning';
$summary['general']['statistics']['warnings']++;
if (!ossim_valid($name, OSS_HOST_NAME, 'illegal:' . _('Hostname'))) {
unset($summary['by_hosts'][$num_line]['warnings']);
$summary['general']['statistics']['warnings']--;
$summary['by_hosts'][$num_line]['status'] = 'error';
$summary['by_hosts'][$num_line]['errors']['Hostname'] = ossim_get_error_clean();
$summary['general']['statistics']['errors']++;
continue;
}
}
//Description
if (!ossim_valid($descr, OSS_NULLABLE, OSS_ALL, 'illegal:' . _('Description'))) {
$summary['by_hosts'][$num_line]['errors']['Description'] = ossim_get_error_clean();
$summary['general']['statistics']['errors']++;
continue;
} else {
if (mb_detect_encoding($descr . ' ', 'UTF-8,ISO-8859-1') == 'UTF-8') {
$descr = mb_convert_encoding($descr, 'HTML-ENTITIES', 'UTF-8');
}
}
//Operating System
$os_pattern = '/' . preg_quote(implode('|', $all_os), '/') . '/';
$os_pattern = str_replace('\\|', '|', $os_pattern);
if (!empty($os) && !preg_match($os_pattern, $os)) {
$warning_msg = _('Operating System unknown');
$summary['by_hosts'][$num_line]['warnings']['Operating System'] = $warning_msg;
$summary['by_hosts'][$num_line]['status'] = 'warning';
$summary['general']['statistics']['warnings']++;
$os = 'Unknown';
}
//Devices Types
$devices = array();
$unallowed_devices = array();
if (!empty($csv_devices)) {
$aux_devices = explode(',', $csv_devices);
if (is_array($aux_devices) && !empty($aux_devices)) {
foreach ($aux_devices as $d_name) {
$d_name = trim($d_name);
if (array_key_exists($d_name, $all_devices)) {
$devices[] = $all_devices[$d_name];
} else {
$unallowed_devices[] = $d_name;
}
}
if (!empty($unallowed_devices)) {
$warning_msg = _('Some devices could not be added (Type and/or subtype unknown)') . ': ' . implode(',', $unallowed_devices);
$summary['by_hosts'][$num_line]['warnings']['Devices'] = $warning_msg;
$summary['by_hosts'][$num_line]['status'] = 'warning';
$summary['general']['statistics']['warnings']++;
}
}
}
//Sensor
if ($is_in_db == FALSE) {
//Only update host sensors with unregistered hosts
if ($import_type == 'hosts' && preg_match('/Sensors/', $csv_headers[5])) {
//Special case: Sensors in CSV file //[Version 3.x.x]
$sensors = array();
$_sensors = explode(',', $param[4]);
if (is_array($_sensors) && !empty($_sensors)) {
$_sensors = array_flip($_sensors);
if (is_array($a_sensors) && !empty($a_sensors)) {
foreach ($a_sensors as $s_id => $s_data) {
if (array_key_exists($s_data['ip'], $_sensors)) {
$sensors[] = $s_id;
}
}
}
}
if (!is_array($sensors) || empty($sensors)) {
$s_error_msg = Session::is_pro() ? _('There is no sensors for this context') : _('There is no sensors for this IP address');
$summary['by_hosts'][$num_line]['errors']['Sensors'] = $s_error_msg;
$summary['general']['statistics']['errors']++;
continue;
}
}
}
/***********************************************************
********** Only for importation from host section **********
***********************************************************/
if ($import_type == 'hosts') {
//FQDNs
if (!ossim_valid($fqdns, OSS_FQDNS, OSS_NULLABLE, 'illegal:' . _('FQDN/Aliases'))) {
$summary['by_hosts'][$num_line]['errors']['FQDN/Aliases'] = ossim_get_error_clean();
$summary['general']['statistics']['errors']++;
continue;
}
//Asset
if (!ossim_valid($asset_value, OSS_DIGIT, 'illegal:' . _('Asset value'))) {
$summary['by_hosts'][$num_line]['errors']['Asset value'] = ossim_get_error_clean();
$summary['general']['statistics']['errors']++;
continue;
}
//Latitude
if (!empty($latitude)) {
if (!ossim_valid(trim($latitude), OSS_NULLABLE, OSS_DIGIT, OSS_DOT, '\\-', 'illegal:' . _('Latitude'))) {
$summary['by_hosts'][$num_line]['errors']['Latitude'] = ossim_get_error_clean();
$summary['general']['statistics']['errors']++;
continue;
}
}
//Longitude
if (!empty($longitude)) {
if (!ossim_valid(trim($longitude), OSS_NULLABLE, OSS_DIGIT, OSS_DOT, '\\-', 'illegal:' . _('Longitude'))) {
$summary['by_hosts'][$num_line]['errors']['Longitude'] = ossim_get_error_clean();
$summary['general']['statistics']['errors']++;
continue;
}
}
}
//Insert/Update host in database
if (count($summary['by_hosts'][$num_line]['errors']) == 0) {
try {
$host = new Asset_host($conn, $host_id);
if ($is_in_db == TRUE) {
$host->load_from_db($conn, $host_id);
$can_i_modify_ips = Asset_host::can_i_modify_ips($conn, $host_id);
} else {
if ($can_i_create_assets == FALSE) {
$n_error_msg = _('Host') . ' ' . $name . ' ' . _("not allowed. You don't have permissions to import this host");
$summary['by_hosts'][$num_line]['errors']['Net'] = $n_error_msg;
$summary['general']['statistics']['errors']++;
continue;
}
}
//Check IPs
if ($can_i_modify_ips == TRUE) {
$aux_ips = explode(',', $csv_ips);
foreach ($aux_ips as $ip) {
$host_ids = Asset_host::get_id_by_ips($conn, $ip, $ctx);
unset($host_ids[$host_id]);
if (!empty($host_ids)) {
$c_error_msg = _('IP') . ' ' . $csv_ips . ' ' . _("not allowed. IP {$ip} already exists for this entity");
$summary['by_hosts'][$num_line]['errors']['IP'] = $c_error_msg;
$summary['general']['statistics']['errors']++;
break;
} else {
$cnd_1 = Session::get_net_where() != '' && !Session::only_ff_net();
$cnd_2 = Asset_host::is_ip_in_cache_cidr($conn, $ip, $ctx, TRUE);
if ($cnd_1 && !$cnd_2) {
$c_error_msg = sprintf(_("Error! The IP %s is not allowed. Please check with your account admin for more information"), $csv_ips);
$summary['by_hosts'][$num_line]['errors']['IP'] = $c_error_msg;
$summary['general']['statistics']['errors']++;
break;
}
}
}
} else {
$c_error_msg = _('Host') . ' ' . $name . ': ' . _("IP address not allowed. IP address cannot be modified");
$summary['by_hosts'][$num_line]['status'] = 'warning';
$summary['general']['warnings']['errors']++;
$summary['by_hosts'][$num_line]['warnings']['IP'] = $c_error_msg;
}
//Setting new values
if (count($summary['by_hosts'][$num_line]['errors']) == 0) {
$host->set_ctx($ctx);
$host->set_name($name);
$host->set_descr($descr);
if ($is_in_db == FALSE) {
if ($can_i_modify_ips == TRUE) {
if (is_array($aux_ips) && !empty($aux_ips)) {
$ips = array();
foreach ($aux_ips as $ip) {
$ips[$ip] = array('ip' => $ip, 'mac' => NULL);
}
$host->set_ips($ips);
}
}
$host->set_sensors($sensors);
}
if (!empty($fqdns)) {
$host->set_fqdns($fqdns);
}
$host->set_external($external);
$host->set_location($latitude, $longitude);
$host->set_asset_value($asset_value);
$host->set_devices($devices);
$host->save_in_db($conn, FALSE);
//Save Operating System
if (!empty($os)) {
Asset_host_properties::save_property_in_db($conn, $host_id, 3, $os, 2);
}
$summary['general']['statistics']['saved']++;
$summary['by_hosts'][$num_line]['data'] = $is_in_db == TRUE ? _('Asset updated') : _('New asset inserted');
//Keep warnings
if ($summary['by_hosts'][$num_line]['status'] != 'warning') {
$summary['by_hosts'][$num_line]['status'] = 'success';
}
}
} catch (Exception $e) {
$summary['by_hosts'][$num_line]['errors']['Database error'] = $e->getMessage();
$summary['general']['statistics']['errors']++;
}
}
}
if ($summary['general']['statistics']['saved'] > 0) {
if ($summary['general']['statistics']['errors'] == 0) {
$summary['general']['status'] = 'success';
$summary['general']['data'] = _('All assets have been successfully imported ');
} else {
$summary['general']['status'] = 'warning';
$summary['general']['data'] = _('Some assets cannot be imported');
}
Util::disable_perm_triggers($conn, FALSE);
try {
Asset_host::report_changes($conn, 'hosts');
} catch (Exception $e) {
Av_exception::write_log(Av_exception::USER_ERROR, $e->getMessage());
}
} else {
$summary['general']['statistics']['errors'] = count($data);
//CSV file is not empty, but all lines are wrong
if (empty($summary['general']['status'])) {
$summary['general']['status'] = 'error';
$summary['general']['data'] = _('Assets cannot be imported');
}
}
@$conn->Execute("REPLACE INTO alienvault.host_net_reference SELECT host.id,net_id FROM alienvault.host, alienvault.host_ip, alienvault.net_cidrs WHERE host.id = host_ip.host_id AND host_ip.ip >= net_cidrs.begin AND host_ip.ip <= net_cidrs.end");
$db->close();
return $summary;
}
$cnd_2 = $_SESSION['_user_vision']['net_where'] && !Session::only_ff_net();
if ($cnd_1 || $cnd_2) {
$db = new ossim_db();
$conn = $db->connect();
$assets .= '<tr>
<td style="text-align:left;width:25mm;font-size:10px;color:#535353;" valign="top">' . "Assets Selected:" . '</td>
<td class="nobborder" style="padding-left:5px;font-size:10px" valign="top">
<table class="w100" cellpadding="0" cellspacing="0">';
if ($_SESSION['_user_vision']['host_where'] && !Session::only_ff_host()) {
$_host_list = Asset_host::get_basic_list($conn);
$hosts = $_host_list[1];
foreach ($hosts as $host) {
$assets .= '<tr><td class="nobborder" style="text-align:left;" valign="top">' . _('Host') . ': ' . $host['name'] . ' [' . $host['ips'] . ']</td></tr>';
}
}
if ($_SESSION['_user_vision']['net_where'] && !Session::only_ff_net()) {
$nets = Asset_net::get_list($conn);
$_net_list = Asset_net::get_list($conn);
$nets = $_net_list[0];
foreach ($nets as $net) {
$assets .= '<tr><td class="nobborder" style="text-align:left;" valign="top">' . _('Net') . ': ' . $net['name'] . ' [' . $net['ips'] . ']</td></tr>';
}
}
$assets .= ' </table>
</td>
</tr>';
$db->close();
} else {
$assets .= '<tr>
<td style="text-align:left;width:25mm;font-size:10px;color:#535353;">' . _('Assets Selected:') . '</td>
<td class="nobborder" style="padding-left:5px;text-align:left;font-size:10px" valign="top">' . _('All Assets') . '</td>
if ($icon != '') {
$image = @imagecreatefromstring($icon);
if (!$image || imagesx($image) > 16 || imagesy($image) > 16) {
$validation_errors['icon'] = _('Image format is not allowed. Allowed only 16x16 PNG images');
}
}
//Validating IPs
$aux_ips = explode(',', $ips_string);
foreach ($aux_ips as $ip) {
$host_ids = Asset_host::get_id_by_ips($conn, $ip, $ctx);
unset($host_ids[$id]);
if (!empty($host_ids)) {
$validation_errors['ip'] = _('Error! IP not allowed.') . " IP {$ip} " . _('already exists for this entity');
break;
} else {
$cnd_1 = Session::get_net_where() != '' && !Session::only_ff_net();
$cnd_2 = Asset_host::is_ip_in_cache_cidr($conn, $ip, $ctx, TRUE);
if ($cnd_1 && !$cnd_2) {
$validation_errors['ip'] = _("Error! IP {$ip} not allowed. Check your asset filter");
break;
}
}
}
//Validating Sensors
if (is_array($sensors) && !empty($sensors)) {
foreach ($sensors as $sensor) {
if (!Av_sensor::is_allowed($conn, $sensor)) {
$validation_errors['sboxs[]'] = _('Error! Host could not be saved because there are unallowed sensors');
}
}
} else {
