public function insert($file, array $callback, $scan_info) { $class = $callback[0]; $method = $callback[1]; $class = Model::factory($class); $this->_handle = fopen($file, 'r'); $headers = fgetcsv($this->_handle, $file); $scan_data = array(); $file = new SplFileObject($file); $file->setFlags(SplFileObject::SKIP_EMPTY); $file->setFlags(SplFileObject::READ_AHEAD); $file->setFlags(SplFileObject::READ_CSV); $file->setCsvControl(",", '"', "\""); $c = 0; foreach ($file as $row) { $c++; if (count($row) === count($headers)) { $scan_data[] = array_combine($headers, $row); $row = array(); } if ($c % $this->insert_threshold == 0) { Logger::msg('info', array('message' => 'flushing ' . $this->insert_threshold . ' rows', "class" => $callback[0], "method" => $callback[1], 'rows_inserted' => $c)); Logger::msg('info', array('memory_usage' => $this->file_size(memory_get_usage()))); $flush = $class->{$method}($scan_data, $scan_info); $scan_data = array(); } } $flush = $class->{$method}($scan_data, $scan_info); $scan_data = array(); Logger::msg('info', array('memory_usage' => $this->file_size(memory_get_usage()))); return $c; }
public function post_url($url, $username, $password, $post_array = NULL) { if (!is_null($post_array)) { $post_string = http_build_query($post_array); } $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); // Timeouts curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 5); curl_setopt($ch, CURLOPT_TIMEOUT, $this->CURLOPT_TIMEOUT); curl_setopt($ch, CURLOPT_LOW_SPEED_TIME, $this->CURLOPT_LOW_SPEED_TIME); curl_setopt($ch, CURLOPT_LOW_SPEED_LIMIT, $this->CURLOPT_LOW_SPEED_LIMIT); if ($this->_request_method === "POST") { curl_setopt($ch, CURLOPT_POST, 1); } if ($post_array) { curl_setopt($ch, CURLOPT_POSTFIELDS, $post_string); } curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE); curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0); curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0); //curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1); curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); curl_setopt($ch, CURLOPT_USERPWD, "{$username}:{$password}"); $curl_result = curl_exec($ch); // Log our curl stats for this run Logger::msg("info", array_merge(array("message" => "curl_stats", "qualys_api_version" => "1"), curl_getinfo($ch))); curl_close($ch); return $curl_result; }
Logger::msg("info", array('report_template_xml' => $report_template_xml)); exit; } // Delete the current report templates in the table for this account $delete_templates = DB::query(Database::DELETE, "DELETE FROM " . REPORT_TEMPLATE_TABLE . " WHERE ACCOUNT = :account")->bind(":account", $account_name)->execute(); Logger::msg("info", array("message" => "deleted report templates", "table" => REPORT_TEMPLATE_TABLE, "account" => $account_name, "rows_deleted" => $delete_templates)); // Put the report template list in the DB $insert = $insert_model->report_templates($report_template_xml, $account_name); Logger::msg("info", array("message" => "report templates inserted for account", "table" => REPORT_TEMPLATE_TABLE, "account" => $account_name, "rows_inserted" => $insert)); // Now lets download our reports defined in config/vulndb.php // Get the reports defined in config/vulndb.php $report_ids = $vdb_config['adr_reports'][$account_name]; // Go get 'em $adr_xml = ""; foreach ($report_ids as $report_id) { Logger::msg("info", array("message" => "downloading asset data report", "report_id" => $report_id, "account" => $account_name, "api_call" => "asset_data_report", "api_version" => 1)); $adr_xml = $api1->asset_data_report($url1, $username, $password, array("template_id" => $report_id)); // Check that we got valid XML if (!$vulndb->is_xml($adr_xml)) { Logger::msg("error", array('message' => 'what I got back from the API call was not XML')); Logger::msg("info", array("asset_data_report_xml" => $adr_xml)); exit; } Logger::msg("info", array("message" => "asset data report download sucessful", "report_id" => $report_id, "account" => $account_name, "api_call" => "asset_data_report", "api_version" => 1)); Logger::msg("info", array("message" => "inserting asset data report into vulnDB", "report_id" => $report_id, "account" => $account_name, "api_call" => "asset_data_report", "api_version" => 1)); $insert = $insert_model->asset_data_report($adr_xml, array("account_name" => $account_name, "report_template_id" => $report_id)); Logger::msg("info", array("message" => "asset data report inserted", "report_id" => $report_id, "account" => $account_name, "rows_inserted" => $insert)); } } Logger::msg("info", array("message" => "asset data report updater is complete"));
if (!is_file($init_file = realpath(dirname(__FILE__)) . "/../init.php")) { echo "Could not find init.php, this file is requied for vulnDB to operate\n"; exit(1); } require $init_file; $insert_model = Model::factory('vulndb_insert'); $vulndb = Model::factory('vulndb_main'); $vdb_config = Config::load('vulndb'); $accounts = $vulndb->getaccounts(); Logger::msg('info', array('message' => 'running scans updater complete')); foreach ($accounts as $account) { $now = date('c'); $account_name = $account['account']; $username = $account['username']; $password = CryptAES::decrypt($account['password']); $url1 = 'https://' . $account['api_url'] . '/msp/'; $url2 = 'https://' . $account['api_url'] . '/api/2.0/fo/'; $api1 = new QualysAPI_v1(); $api2 = new QualysAPI_v2($url2, $username, $password); Logger::msg('info', array('message' => 'polling scans', 'account' => $account_name)); $scanlist_v2 = $api2->pollscans(date('Y-m-d', strtotime('-15 day')), array('state' => 'Running')); $scanlist_v1 = $api1->scan_running_list($url1, $username, $password); // It appears Qualys isnt returning XML when there are no scans anymore :( if (!$scanlist_v1) { continue; } $insert = $insert_model->running_scans($scanlist_v1, $scanlist_v2, $account_name); Logger::msg('info', array('message' => "running scans updated", 'account' => $account_name)); } Logger::msg('info', array('message' => 'running scans updater complete'));
public function insert($table, $data) { $fields = array_keys(reset($data)); $insert = DB::insert($table, $fields); $c = 0; foreach ($data as $d) { $c++; $insert->values($d); if ($c % 500 === 0) { $insert->execute(); $insert->reset_values(); } } $insert->execute(); Logger::msg("info", array("message" => "DB insert complete", "rows_inserted" => $c, "table" => $table, "class_name" => __CLASS__, "method" => __METHOD__)); return $c; }
/** Deprecated on 2014-01-03 --- can be removed after testing **/ public function post_url_old($url, $post_array, $header_array, $options = NULL) { $post_string = http_build_query($post_array); $ch = curl_init($url); // Set our tmp cookie files curl_setopt($ch, CURLOPT_COOKIEJAR, $this->cookie_file); curl_setopt($ch, CURLOPT_COOKIEFILE, $this->cookie_file); // Timeouts curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 500); curl_setopt($ch, CURLOPT_LOW_SPEED_LIMIT, 100); curl_setopt($ch, CURLOPT_LOW_SPEED_TIME, 60); // Don't return the header curl_setopt($ch, CURLOPT_HEADER, FALSE); if ($post_array) { curl_setopt($ch, CURLOPT_POST, 1); curl_setopt($ch, CURLOPT_POSTFIELDS, $post_string); } curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_setopt($ch, CURLOPT_HTTPHEADER, $header_array); curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, 0); curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, 0); if (isset($options['return_stream'])) { if (isset($options['return_file'])) { if (!isset($options['output_filename'])) { throw new Exception("Output_filename must be set, and must be a full path"); } $fp = fopen($options['output_filename'], 'w+'); } else { $fp = fopen('qapi://memory', 'r+'); } curl_setopt($ch, CURLOPT_FILE, $fp); $curl_result = curl_exec($ch); fclose($fp); } else { $curl_result = curl_exec($ch); } $raw_headers = substr($curl_result, 0, strpos($curl_result, "\r\n\r\n")); $body = substr($curl_result, strpos($curl_result, "\r\n\r\n")); $result = $body; $raw_header_array = explode("\r\n", $raw_headers); $http_code = array_shift($raw_header_array); foreach ($raw_header_array as $header_line) { $key = strtoupper(trim(substr($header_line, 0, strpos($header_line, ":")))); $val = trim(substr($header_line, strpos($header_line, ":") + 1)); $headers[$key] = $val; } // Log our curl stats for this run Logger::msg("info", array_merge(array("message" => "curl_stats"), curl_getinfo($ch))); // Close the curl connection curl_close($ch); return $result; }
} ini_set('memory_limit', '512M'); require $init_file; Logger::msg("info", array('message' => "KB updater starting")); $vdb_config = Config::load('vulndb'); // Name of the account we will use to pull back in the KB // Define it config/vulndb $kb_account = $vdb_config['kb_account']; $vulndb = Model::factory('vulndb_main'); $insert = Model::factory('vulndb_insert'); $account_info = $vulndb->getaccountinfo($kb_account); $username = $account_info['username']; $password = CryptAES::decrypt($account_info['password']); $url1 = $account_info['url1']; $api1 = new QualysAPI_v1(); Logger::msg('info', array('message' => "Qualys KB download beginning", 'api_call' => 'get_qualys_kb', 'api_version' => '1', 'kb_account' => $kb_account, 'account_username' => $username)); $KB_XML = $api1->get_qualys_kb($url1, $username, $password); Logger::msg('info', array('message' => "Qualys KB download complete", 'api_call' => 'get_qualys_kb', 'api_version' => '1')); // check the XML if (!$vulndb->is_xml($KB_XML)) { Logger::msg("error", array('message' => 'what I got back from the API call was not XML')); Logger::msg("info", array('kb_xml' => $KB_XML)); exit; } // If we got good XML back, go ahead an truncate the current KB $truncate = DB::query(Database::DELETE, "TRUNCATE " . MAIN_QUALYS_KB_TABLE)->execute(); Logger::msg("info", array('message' => "Qualys KB table, " . MAIN_QUALYS_KB_TABLE . " truncated")); Logger::msg("info", array('message' => "KB insert beginning")); $go = $insert->kb($KB_XML); Logger::msg("info", array('message' => "KB insert complete", "rows_inserted" => $go)); unset($api1);
$insert_details = $insert_model->scan_details($scan_to_get, array('ACCOUNT' => $account_name, "DATE_ENTERED" => $now)); } else { Logger::msg('info', array('account' => $account_name, 'message' => 'unable to insert scan into vulnDB', 'scan_id' => $scanid, 'scan_title' => $scantitle, 'scan_status', $scanstatus)); } } elseif ($scanstatus = 'RUNNING') { // We can't pull in running scans, nor do we want to put that entry into the db continue; } else { Logger::msg('info', array('account' => $account_name, 'message' => 'scan status did not match $scantypestoget and will not be pulled in', 'scan_id' => $scanid, 'scan_title' => $scantitle, 'scan_status' => $scanstatus)); $insert_details = $insert_model->scan_details($scan_to_get, array("ACCOUNT" => $account_name, "DATE_ENTERED" => $now)); } } } else { Logger::msg('info', array('account' => $account_name, 'message' => "no scans found for this account. moving onto the next one")); } // Delete AGs for account so we always have a fresh copy // If you want to keep a trail for the AG's, just comment this out $deleted_ags = DB::query(Database::DELETE, "DELETE FROM " . MAIN_AG_TABLE . " WHERE ACCOUNT=:account")->bind(':account', $account_name)->execute(); // Pull in the asset groups for the account Logger::msg('info', array('message' => 'downloading asset groups', 'api_call' => 'get_asset_groups', 'api_version' => 1)); $ags = $api1->get_asset_groups($url1, $username, $password); // Put AG's into vulnDB $insert = $insert_model->ags($ags, $account_name); Logger::msg('info', array('message' => 'asset groups successfully entered into vulndb', 'account' => $account_name)); // release the api classes unset($api1); unset($api2); } // end foreaach($accounts...) Logger::msg('info', array('message' => 'Ending vulnDB updater'));
* **/ if (!is_file($init_file = realpath(dirname(__FILE__)) . "/../init.php")) { echo "Could not find init.php, this file is requied for vulnDB to operate\n"; exit(1); } require $init_file; $insert_model = Model::factory('vulndb_insert'); $vulndb = Model::factory('vulndb_main'); $vdb_config = Config::load('vulndb'); Logger::msg('info', array('message' => 'Starting the asset data report updater')); $accounts = $vulndb->getaccounts(); foreach ($accounts as $account) { $now = date('c'); $account_name = $account['account']; $username = $account['username']; $password = CryptAES::decrypt($account['password']); $url1 = 'https://' . $account['api_url'] . '/msp/'; $url2 = 'https://' . $account['api_url'] . '/api/2.0/fo/'; $api1 = new QualysAPI_v1(); $api2 = new QualysAPI_v2($url2, $username, $password); // Delete AGs for account so we always have a fresh copy // If you want to keep a trail for the AG's, just comment this out $deleted_ags = DB::query(Database::DELETE, "DELETE FROM " . MAIN_AG_TABLE . " WHERE ACCOUNT=:account")->bind(':account', $account_name)->execute(); // Pull in the asset groups for the account Logger::msg('info', array('message' => 'downloading asset groups', 'api_call' => 'get_asset_groups', 'api_version' => 1, 'account_name' => $account_name)); $ags = $api1->get_asset_groups($url1, $username, $password); // Put AG's into vulnDB $insert = $insert_model->ags($ags, $account_name); Logger::msg('info', array('message' => 'asset groups successfully entered into vulndb', 'account_name' => $account_name)); }