public function crowl_products($store_id, $type = 'all', $cron_log_id = '0')
{
// A list of existing APIs
$valid_types = get_market_lookup(FALSE, TRUE);
$valid_types['all'] = TRUE;
if (!isset($valid_types[$type])) {
throw new InvalidArgumentException($type . ' is not a valid marketplace.', 1301);
}
// argument not in valid set
// Crawl using these APIs
$apis = $type === 'all' ? array_keys($valid_types) : array($type);
$date_stamp = date('Y-m-d H:i:s');
$offset = 0;
// Get the total number of records
$where = array('store_id' => (int) $store_id, 'is_tracked' => 1, 'is_archived' => 0, 'status' => 1, 'length(upc_code) >=' => 10);
$result_count = $this->db->select('count(*) as total_record')->where($where)->get($this->_table_products)->row();
$total_records = $result_count ? $result_count->total_record : 0;
// Crawl the products 100 at a time
while ($total_records > $offset) {
$results = $this->db->where($where)->group_by('upc_code')->limit(100, $offset)->get($this->_table_products)->result();
// TODO: do a fallback strategy for keyword search: if no UPC, then use other data
//This scraper must:
// 1) find products based upon UPC code
// 2) if not successful above, find products based upon manufacturer + manufacturer part number
// 3) if not successful above, find products based upon manufacturer + product name
for ($i = 0, $n = count($results); $i < $n; $i++) {
$upc = trim($results[$i]->upc_code);
if ($upc) {
if (empty($this->globalArray[$type][$upc])) {
$this->globalArray[$type][$upc] = true;
$id = $results[$i]->id;
$upcs = array($id => $upc);
$price_floors = array($id => array('floor_price' => $results[$i]->price_floor));
// Lookup UPC using proper API
foreach ($apis as $api) {
$lookup_func = $api . '_lookup';
try {
$this->{$lookup_func}($upcs, $price_floors, $cron_log_id, $store_id);
} catch (Exception $e) {
email_alertToTeam($lookup_func, $e->getMessage());
}
}
}
}
}
$offset += 100;
}
$this->db->where('id', (int) $store_id)->update($this->_table_store, array('tracked_at' => $date_stamp));
return true;
}
private function _prep_pricing_overview()
{
$this->data->my = 'pricingOverview';
$this->data->report_chart = 'na';
// Get the last crawl data
$this->data->last_crawl = $this->crawl_data_m->last_crawl();
$this->crawl_range = $this->crawl_data_m->last_crawl_range();
$this->data->number_of_merchants = getNumberOfMerchants($this->report_info->store_id, $this->crawl_range['from'], $this->crawl_range['to']);
$this->data->last_tracked_arr = $this->Report->last_tracked_image($this->crawl_range['from']);
$this->data->last_tracked_date = trackingDateFormat($this->crawl_range['from']);
// Calculate the overview statistics
$this->data->products_monitored = $this->Product->getProductsMonitoredCount($this->report_info->store_id);
$this->data->total_violations = $this->Violator->countViolatedProducts($this->report_info->store_id);
$this->data->marketplace_products = array();
$this->data->market_violations = array();
$markets = array_keys(get_market_lookup(TRUE));
foreach ($markets as $market) {
$crawl_info = !empty($this->data->last_crawl[$market]) ? $this->data->last_crawl[$market] : FALSE;
if ($crawl_info) {
$from = $crawl_info->start_datetime;
$to = $crawl_info->end_datetime;
$marketplace_products = $this->merchant_products_m->getCountByMarketplace($this->report_info->store_id, $market, $from, $to);
if (!empty($marketplace_products)) {
$market_violations = $this->Violator->getViolatedMarkets($this->report_info->store_id, '', $market, $from, $to);
$this->data->marketplace_products[] = $marketplace_products[0];
$this->data->market_violations[$market] = $market_violations[$market];
}
}
}
$this->data->request_info = array('fromDate' => strtotime($this->crawl_range['from']), 'toDate' => strtotime($this->crawl_range['to']), 'time_frame' => 1, 'product_ids' => array());
$this->data->gData = array('type' => 'pie', 'width' => '220', 'height' => '150', 'googleData' => array(array('State', 'Count'), array('Non Violation', (int) max($this->data->products_monitored - $this->data->total_violations, 0)), array('Violation', (int) $this->data->total_violations)));
// Separate marketplaces from retailers
$this->data->marketplaces = array();
$this->data->retailers = array();
$this->data->violatedMarketplaces = array();
$this->data->violatedRetailers = array();
for ($i = 0, $n = count($this->data->marketplace_products); $i < $n; $i++) {
$name = $this->data->marketplace_products[$i]['marketplace'];
if ($this->data->marketplace_products[$i]['is_retailer'] === '1') {
$merchant = $this->merchant_products_m->getMerchantDetailsByMarketplace($name);
if (isset($merchant[0]['id'])) {
$this->data->marketplace_products[$i] = array_merge($this->data->marketplace_products[$i], $merchant[0]);
$this->data->retailers[] = $this->data->marketplace_products[$i];
if (!empty($this->data->market_violations[$name])) {
$this->data->violatedRetailers[$name] = TRUE;
}
}
} else {
$this->data->marketplaces[] = $this->data->marketplace_products[$i];
if (!empty($this->data->market_violations[$name])) {
$this->data->violatedMarketplaces[$name] = TRUE;
}
}
}
$this->data->Data->retailers = $this->data->retailers;
$this->data->Data->marketplaces = $this->data->marketplaces;
//print_r($this->data); exit;
}
;
}
/*For Google Charts*/
var repChartContainer = null;
var data = null;
var chart = null;
var chartdata = null;
var myView = null;
var options = null;
var globalData = getGoogleData();
google.load("visualization", "1", {packages:["corechart"]});
google.setOnLoadCallback(drawChart);
<?php
//if( ! empty($gData['data']['size'])):
?>
google.setOnLoadCallback(drawGoogleChart);
var googleData = getGoogleData();
var marketLookup = <?php
echo json_encode(get_market_lookup());
?>
;
<?php
//endif;
}
?>
</script>
/**
* New dashboard for violation area.
*
* Formerly known as violation_dashboard()
*
* @author Christophe
* @param int $reportId
*/
public function index($reportId = FALSE)
{
$this->load->model("crawl_data_m", 'Crawl');
$this->load->model("merchant_products_m", 'MProducts');
$this->load->model("Products_m", 'Product');
$this->load->model('violator_m', 'Violator');
$this->load->model("report_m", 'Report');
$this->load->model("store_m", 'Store');
$this->load->model("Users_m", 'User');
$this->last_crawl = $this->Crawl->last_crawl();
$this->crawl_range = $this->Crawl->last_crawl_range();
$this->data->my = 'pricingOverview';
$this->record_per_page = $this->config->item("record_per_page");
// tell the view this is the pricing overview report
$this->data->report_name = 'Pricing Overview';
$this->data->report_type = 'pricingoverview';
$this->data->file_name = str_replace(' ', '_', 'Pricing Overview ' . date('Y-m-d'));
if ($reportId) {
$this->_set_defaults($reportId);
}
$this->data->number_of_merchants = getNumberOfMerchants($this->store_id, $this->crawl_range['from'], $this->crawl_range['to']);
$this->data->last_tracked_arr = $this->Report->last_tracked_image($this->crawl_range['from']);
$this->data->last_tracked_date = trackingDateFormat($this->crawl_range['from']);
// calculate the overview statistics
$this->data->products_monitored = $this->Product->getProductsMonitoredCount($this->store_id);
$this->data->total_violations = $this->Violator->countViolatedProducts($this->store_id);
$this->data->marketplace_products = array();
$this->data->market_violations = array();
$markets = array_keys(get_market_lookup(TRUE, TRUE));
foreach ($markets as $market) {
$crawl_info = !empty($this->last_crawl[$market]) ? $this->last_crawl[$market] : FALSE;
// using global crawl time - change by Christophe on 9/2/2015
if ($this->config->item('environment') == 'local') {
$crawl_info = TRUE;
}
if ($crawl_info) {
if ($this->config->item('environment') == 'local') {
$from = '2015-08-01 00:00:00';
$to = '2015-08-02 00:00:00';
} else {
//$from = date('Y-m-d H:i:s', strtotime('-24 hours'));
//$to = date('Y-m-d H:i:s');
$from = $crawl_info->start_datetime;
$to = $crawl_info->end_datetime;
}
$marketplace_products = $this->MProducts->getCountByMarketplace($this->store_id, $market, $from, $to);
if (!empty($marketplace_products)) {
$this->data->marketplace_products[] = $marketplace_products[0];
$market_violations = $this->Violator->getViolatedMarkets($this->store_id, '', $market, $from, $to);
//$market_violations = $CI->Violator->getMarketViolations($this->store_id, '', $market);
$this->data->market_violations[$market] = $market_violations[$market];
}
}
}
//for saving reports - not necessary to provide info for overview
$this->data->report_where = array('report_type' => 'pricingoverview');
$this->data->gData = array('type' => 'pie', 'width' => '220', 'height' => '150', 'googleData' => array(array('State', 'Count'), array('Non Violation', (int) max($this->data->products_monitored - $this->data->total_violations, 0)), array('Violation', (int) $this->data->total_violations)));
// separate marketplaces from retailers
$this->data->marketplaces = array();
$this->data->retailers = array();
$this->data->violatedMarketplaces = array();
$this->data->violatedRetailers = array();
for ($i = 0, $n = count($this->data->marketplace_products); $i < $n; $i++) {
$name = $this->data->marketplace_products[$i]['marketplace'];
if ($this->data->marketplace_products[$i]['is_retailer'] === '1') {
$merchant = $this->MProducts->getMerchantDetailsByMarketplace($name);
if (isset($merchant[0]['id'])) {
$this->data->marketplace_products[$i] = array_merge($this->data->marketplace_products[$i], $merchant[0]);
$this->data->retailers[] = $this->data->marketplace_products[$i];
if (!empty($this->data->market_violations[$name])) {
$this->data->violatedRetailers[$name] = TRUE;
}
}
} else {
$this->data->marketplaces[] = $this->data->marketplace_products[$i];
if (!empty($this->data->market_violations[$name])) {
$this->data->violatedMarketplaces[$name] = TRUE;
}
}
}
usort($this->data->marketplaces, function ($a, $b) {
return strtolower($a['marketplace']) > strtolower($b['marketplace']);
});
usort($this->data->retailers, function ($a, $b) {
return strtolower($a['marketplace']) > strtolower($b['marketplace']);
});
$this->data->totalMarketplaces = count($this->data->marketplaces);
$this->data->totalRetailers = count($this->data->retailers);
}
/**
* Used with Reportinfo.php - report_overview()
*
* @author Christophe
* @param unknown_type $storeId
* @param unknown_type $append
* @param unknown_type $marketplace
* @return multitype:number
*/
function getViolatedMarketsNoDateRange($storeId, $append = '_violations', $marketplace = 'all')
{
$ret = array();
$markets = $marketplace == 'all' ? array_keys(get_market_lookup(TRUE)) : array($marketplace);
$sql = "\n \t\tSELECT COUNT(cpl.id) as count\n \t\tFROM {$this->_table_crowl_product_list} cpl\n \t\tLEFT JOIN {$this->_table_products} p ON p.upc_code = cpl.upc\n \t\tWHERE\n \t\tp.store_id IN (" . getStoreIdList($storeId, TRUE) . ")\n \t\tAND cpl.violated = 1\n \t\tAND cpl.marketplace = ?\n\t\t";
foreach ($markets as $market) {
$params = array($market);
$count = $this->db->query($sql, $params)->row();
//echo $this->db->last_query()."\n";
$ret[$market . $append] = $count ? (int) $count->count : 0;
}
return $ret;
}