public function post_commands()
{
$params = Input::param();
require_once 'AWSSDKforPHP/sdk.class.php';
$ddb = new AmazonDynamoDB();
$ddb->set_region('dynamodb.ap-northeast-1.amazonaws.com');
$body = $params['body'];
$ret = $ddb->put_item(array('TableName' => 'watasync2b', 'Item' => array('id' => array(AmazonDynamoDB::TYPE_STRING => (string) uniqid()), 'command' => array(AmazonDynamoDB::TYPE_STRING => (string) $body), 'created_at' => array(AmazonDynamoDB::TYPE_NUMBER => (string) time()), 'updated_at' => array(AmazonDynamoDB::TYPE_NUMBER => (string) time()))));
$responce = array("hoge" => "hoge");
$this->response($responce, 200);
}
/**
* @covers mychaelstyle\datastore\providers\AmazonDynamoDB::write
* @covers mychaelstyle\datastore\providers\AmazonDynamoDB::batchWrite
* @covers mychaelstyle\datastore\providers\AmazonDynamoDB::get
* @covers mychaelstyle\datastore\providers\AmazonDynamoDB::batchGet
* @covers mychaelstyle\datastore\providers\AmazonDynamoDB::connect
*/
public function testFlow()
{
$t = time();
$expected = array('tests' => array(array('id' => 'AAAA0001', 'updated_at' => (int) $t, 'body' => 'This is test!')));
// write
$this->object->write('tests', $expected['tests'][0]);
// get
$result = $this->object->get('tests', array('id' => 'AAAA0001'));
$this->assertEquals($expected['tests'][0], $result);
// remove
$this->object->remove('tests', array('id' => 'AAAA0001'));
$result = $this->object->get('tests', array('id' => 'AAAA0001'));
$this->assertNull($result);
// batch write
$this->object->batchWrite($expected);
// batchGet
$result = $this->object->batchGet(array('tests' => array(array('id' => 'AAAA0001'))));
$this->assertEquals($expected, $result);
// batch remove
$result = $this->object->batchRemove(array('tests' => array(array('id' => 'AAAA0001'))));
$result = $this->object->get('tests', array('id' => 'AAAA0001'));
$this->assertNull($result);
}
private function _retailer_lookup_deprecated($api, $search_url, $upcs, $floor_price, $cron_log_id, $store_id)
{
foreach ($upcs as $key => $upc) {
}
// only one index
$price_floor = $floor_price[$key]['floor_price'];
$retailer_data = $this->_spider_search($upc, $api, $search_url);
if (!isset($retailer_data[0])) {
$this->updateUPCFlag($upc, $api, '0');
return;
}
$this->myDB = new mydb();
$avg_price = 0;
$n = 0;
$violationFlag = false;
$uniquArr = array();
foreach ($retailer_data as $item) {
$this->stats[$api]['data_found']++;
$crawled_product = array();
$violation = array();
$author = $this->getMerchantNameForKey($api, $upc, $api, $api, $api, $search_url);
if (empty($author)) {
// assert dummy value
$author = "No name found.";
}
$stQuery = "UPDATE cron_log set google_count=(google_count+1), last_UPC='{$upc}' WHERE id= '{$cron_log_id}' ";
$this->myDB->simpleQuery($stQuery);
$hashKey = $author . '#' . $upc;
$crawled_product['um'] = $hashKey;
$crawled_product['dt'] = time();
$crawled_product['ap'] = $price_floor;
$crawled_product['t'] = isset($item['product.name']) ? $item['product.name'] : '';
$crawled_product['mu'] = isset($item['seller.aboutus']) ? $item['seller.aboutus'] : '';
$crawled_product['l'] = isset($item['product.url']) ? $item['product.url'] : '';
$crawled_product['ar'] = $api;
$crawled_product['mil'] = isset($item['seller.logo']) ? $item['seller.logo'] : '';
$crawled_product['il'] = isset($item['product.image_url']) ? $item['product.image_url'] : '';
$crawled_product['mpo'] = isset($item['product.price_listed']) ? $this->parseDollarAmount($item['product.price_listed']) : 0;
$crawled_product['msp'] = isset($item['product.shipping']) ? $item['product.shipping'] : 0;
if ($crawled_product['mpo']) {
$this->stats[$api]['price_found']++;
$avg_price = $this->_new_avg($avg_price, $crawled_product['mpo']);
// Get the price from the last crawl
$dynamo = new AmazonDynamoDB();
$lastRecordResponse = $dynamo->query(array('TableName' => $this->_dynamo_products_trends, 'HashKeyValue' => array(AmazonDynamoDB::TYPE_STRING => $crawled_product['um']), 'ConsistentRead' => true, 'Limit' => 1, 'ScanIndexForward' => false));
$insert_response = $this->amzdb->insertData($this->_dynamo_products_trends, $crawled_product, $api);
if (isset($insert_response->status) && $insert_response->status == 200) {
$this->insertUPCMerchant($author, $upc, $api, $crawled_product['mpo'], $api, $api);
}
// get the corresponding crowlMerchantName row
$crowlMerchantName = $this->Crawl_data->crowlMerchantBySellerID($api);
if (empty($crowlMerchantName)) {
log_message('error', __FILE__ . ' Crowl_m::' . $api . '_lookup() Line ' . __LINE__ . ': crowl_merchant_name record not found for seller ID ' . $api);
}
$crowlMerchantNameID = isset($crowlMerchantName->id) ? $crowlMerchantName->id : 0;
$violatedPrice = (double) $crawled_product['mpo'];
$dataVio = $this->Products->get_products_by_floor($upc, $violatedPrice, $store_id);
if ($dataVio) {
$violation['um'] = $hashKey;
$violation['dt'] = $crawled_product['dt'];
$violation['ss'] = date('Ymd', $violation['dt']) . '/' . md5($hashKey . $violation['dt']) . '.png';
$this->updateViolationSummary($dataVio);
$this->Violator->updatePriceViolator($crowlMerchantNameID, $upc, 1, $crawled_product['dt']);
$lastCrawlPrice = (double) 0;
$hashKey = null;
$rangeKey = null;
if ($lastRecordResponse->isOK()) {
$mpo = isset($lastRecordResponse->body->Items->mpo->N) ? (double) $lastRecordResponse->body->Items->mpo->N : (double) 0;
// Merchant Price Offered
$lastCrawlPrice = $mpo;
$hashKey = isset($lastRecordResponse->body->Items->um->S) ? $lastRecordResponse->body->Items->um->S : null;
$rangeKey = isset($lastRecordResponse->body->Items->dt->N) ? $lastRecordResponse->body->Items->dt->N : null;
}
// Check if the price has changed
if ((double) $lastCrawlPrice != (double) $violatedPrice) {
if (!isset($uniquArr[$crawled_product['l']])) {
$uniquArr[$crawled_product['l']] = $violation['ss'];
$this->addScreenShot($crawled_product['l'], $violation['ss'], false, $violatedPrice);
} else {
$violation['ss'] = $uniquArr[$crawled_product['l']];
}
} else {
$takeNewScreenShot = true;
// Get the violation screen shot
if (!empty($hashKey) and !empty($rangeKey)) {
$lastViolationResponse = $dynamo->query(array('TableName' => $this->_dynamo_violations, 'HashKeyValue' => array(AmazonDynamoDB::TYPE_STRING => (string) $hashKey), 'RangeKeyCondition' => array('ComparisonOperator' => AmazonDynamoDB::CONDITION_EQUAL, 'AttributeValueList' => array(array(AmazonDynamoDB::TYPE_NUMBER => (string) $rangeKey))), 'ConsistentRead' => 'true'));
if ($lastViolationResponse->isOK() and $lastViolationResponse->body->Count == 1) {
if (isset($lastViolationResponse->body->Items->ss->S)) {
if (@fopen(get_instance()->config->item('s3_cname') . 'stickyvision/violations/' . $lastViolationResponse->body->Items->ss->S, 'r')) {
$violation['ss'] = $lastViolationResponse->body->Items->ss->S;
$takeNewScreenShot = false;
}
}
}
}
if ($takeNewScreenShot === true) {
$this->addScreenShot($crawled_product['l'], $violation['ss'], false, $violatedPrice);
}
}
$this->amzdb->insertData($this->_dynamo_violations, $violation, $api);
$violationFlag = true;
unset($dataVio);
} else {
// update price violators for products not in violation
$this->Violator->updatePriceViolator($crowlMerchantNameID, $upc, 0, $crawled_product['dt']);
}
$n++;
} else {
email_alertToTeam($api . '_lookup', 'Product data found, but price not found for UPC ' . $upc);
}
unset($retailer_data, $crawled_product, $violation);
}
$this->myDB->close();
$this->updateUPCFlag($upc, $api, '1');
}
<?php
// If necessary, reference the sdk.class.php file.
// For example, the following line assumes the sdk.class.php file is
// in an sdk sub-directory relative to this file
require_once dirname(__FILE__) . '/vendor/amazonwebservices/aws-sdk-for-php/sdk.class.php';
// Instantiate the class.
$dynamodb = new AmazonDynamoDB(array('key' => getenv('AWS_ACCESS_KEY'), 'secret' => getenv('AWS_SECRET_KEY')));
$dynamodb->set_region('dynamodb.' . getenv('AWS_REGION') . '.amazonaws.com');
$fourteen_days_ago = date('Y-m-d H:i:s', strtotime("-14 days"));
$response = $dynamodb->query(array('TableName' => 'Reply', 'HashKeyValue' => array(AmazonDynamoDB::TYPE_STRING => 'Amazon DynamoDB#DynamoDB Thread 2'), 'RangeKeyCondition' => array('ComparisonOperator' => AmazonDynamoDB::CONDITION_GREATER_THAN_OR_EQUAL, 'AttributeValueList' => array(array(AmazonDynamoDB::TYPE_STRING => $fourteen_days_ago)))));
header('content-type: text/plain');
// Response code 200 indicates success
print_r($response);
?>
/**
* Attempts to lock the specified field
* if it is a 0. Returns the the old value and previous
* timestamp on success, FALSE if the lock can't be obtained
*
* @param mixed $id
* @param mixed $field
* @param mixed $lock
* @param mixed $force
* @return mixed
*/
function lock_lock($id, $field = 'lock', $lock = true, $force = false)
{
$r = new ddbRequest($this->TableName);
if ($lock) {
$expected_value = 0;
$new_value = 1;
} else {
$expected_value = 1;
$new_value = 0;
}
$r->setKey($id);
$r->setReturnValues(AmazonDynamoDB::RETURN_UPDATED_OLD);
if (!$force) {
$r->setExpected(array($field => $expected_value));
}
$r->setUpdates(array($field => $new_value, $field . '_ts' => time()), AmazonDynamoDB::ACTION_PUT);
$params = $r->getParams();
$this->response = $this->dynamodb->update_item($params);
$status = $this->response->status;
if ($status == '200') {
return ddbUtil::parse_item($this->response->body);
} else {
return false;
}
}
function create_table()
{
// Instantiate the class.
$dynamodb = new AmazonDynamoDB();
$response = $dynamodb->create_table(array('TableName' => $this->tableName, 'KeySchema' => array('HashKeyElement' => array('AttributeName' => $this->primaryKey, 'AttributeType' => AmazonDynamoDB::TYPE_NUMBER)), 'ProvisionedThroughput' => array('ReadCapacityUnits' => 5, 'WriteCapacityUnits' => 5)));
// Check for success...
if ($response->isOK()) {
// continue
} else {
echo '# A ERROR HAS OCCURED<br />';
print_r($response);
return false;
}
####################################################################
# Sleep and poll until the table has been created
$count = 0;
do {
sleep(1);
$count++;
$response = $dynamodb->describe_table(array('TableName' => $this->tableName));
} while ((string) $response->body->Table->TableStatus !== 'ACTIVE');
return true;
}
public static function ensureSession()
{
$session_id = session_id();
if (empty($session_id)) {
// Expand session lifetime
ini_set("session.gc_maxlifetime", 24 * 60 * 60);
// Instantiate the Amazon DynamoDB client.
// REMEMBER: You need to set 'default_cache_config' in your config.inc.php.
$dynamodb = new AmazonDynamoDB();
// Register the DynamoDB Session Handler.
$handler = $dynamodb->register_session_handler(array('table_name' => 'internationalize-sessions', 'session_lifetime' => 0, 'session_locking' => false));
session_start();
}
}
* You've verified that your PHP environment passes the SDK Compatibility Test.
* You've already added your credentials to your config.inc.php file, as per the
instructions in the Getting Started Guide.
* You've already created a table in Amazon DynamoDB called "sessions-test" with
a string primary key called "id".
TO RUN:
* Run this file on your web server by loading it in your browser. It will
generate HTML output.
*/
// Include the SDK
require_once dirname(dirname(__FILE__)) . '/sdk.class.php';
// Instantiate a DynamoDB client
$dynamodb = new AmazonDynamoDB();
// Instantiate, configure, and register the session handler
$session_handler = $dynamodb->register_session_handler(array('table_name' => 'sessions-test', 'lifetime' => 3600));
// Open the session
session_start();
// If the logout flag is set, do a logout instead
if (isset($_GET['logout'])) {
// Destroy the session
session_destroy();
// Refresh this page to show the login form
header('Location: html-dynamodb_sessions_login.php');
exit(0);
}
// Simulate authentication by checking for username and password
if (isset($_POST['username']) && isset($_POST['password'])) {
$username = trim($_POST['username']);
<?php
// If necessary, reference the sdk.class.php file.
// For example, the following line assumes the sdk.class.php file is
// in an sdk sub-directory relative to this file
require_once dirname(__FILE__) . '/vendor/amazonwebservices/aws-sdk-for-php/sdk.class.php';
// Instantiate the class
$dynamodb = new AmazonDynamoDB(array('key' => getenv('AWS_ACCESS_KEY'), 'secret' => getenv('AWS_SECRET_KEY')));
$dynamodb->set_region('dynamodb.' . getenv('AWS_REGION') . '.amazonaws.com');
####################################################################
# Setup some local variables for dates
$one_day_ago = date('Y-m-d H:i:s', strtotime("-1 days"));
$seven_days_ago = date('Y-m-d H:i:s', strtotime("-7 days"));
$fourteen_days_ago = date('Y-m-d H:i:s', strtotime("-14 days"));
$twenty_one_days_ago = date('Y-m-d H:i:s', strtotime("-21 days"));
####################################################################
// Set up batch requests
$queue = new CFBatchRequest();
$queue->use_credentials($dynamodb->credentials);
// Add items to the batch
$dynamodb->batch($queue)->put_item(array('TableName' => 'ProductCatalog', 'Item' => array('Id' => array(AmazonDynamoDB::TYPE_NUMBER => '101'), 'Title' => array(AmazonDynamoDB::TYPE_STRING => 'Book 101 Title'), 'ISBN' => array(AmazonDynamoDB::TYPE_STRING => '111-1111111111'), 'Authors' => array(AmazonDynamoDB::TYPE_ARRAY_OF_STRINGS => array('Author1')), 'Price' => array(AmazonDynamoDB::TYPE_NUMBER => '2'), 'Dimensions' => array(AmazonDynamoDB::TYPE_STRING => '8.5 x 11.0 x 0.5'), 'PageCount' => array(AmazonDynamoDB::TYPE_NUMBER => '500'), 'InPublication' => array(AmazonDynamoDB::TYPE_NUMBER => '1'), 'ProductCategory' => array(AmazonDynamoDB::TYPE_STRING => 'Book'))));
$dynamodb->batch($queue)->put_item(array('TableName' => 'ProductCatalog', 'Item' => array('Id' => array(AmazonDynamoDB::TYPE_NUMBER => '102'), 'Title' => array(AmazonDynamoDB::TYPE_STRING => 'Book 102 Title'), 'ISBN' => array(AmazonDynamoDB::TYPE_STRING => '222-2222222222'), 'Authors' => array(AmazonDynamoDB::TYPE_ARRAY_OF_STRINGS => array('Author1', 'Author2')), 'Price' => array(AmazonDynamoDB::TYPE_NUMBER => '20'), 'Dimensions' => array(AmazonDynamoDB::TYPE_STRING => '8.5 x 11.0 x 0.8'), 'PageCount' => array(AmazonDynamoDB::TYPE_NUMBER => '600'), 'InPublication' => array(AmazonDynamoDB::TYPE_NUMBER => '1'), 'ProductCategory' => array(AmazonDynamoDB::TYPE_STRING => 'Book'))));
$dynamodb->batch($queue)->put_item(array('TableName' => 'ProductCatalog', 'Item' => array('Id' => array(AmazonDynamoDB::TYPE_NUMBER => '103'), 'Title' => array(AmazonDynamoDB::TYPE_STRING => 'Book 103 Title'), 'ISBN' => array(AmazonDynamoDB::TYPE_STRING => '333-3333333333'), 'Authors' => array(AmazonDynamoDB::TYPE_ARRAY_OF_STRINGS => array('Author1', 'Author2')), 'Price' => array(AmazonDynamoDB::TYPE_NUMBER => '2000'), 'Dimensions' => array(AmazonDynamoDB::TYPE_STRING => '8.5 x 11.0 x 1.5'), 'PageCount' => array(AmazonDynamoDB::TYPE_NUMBER => '600'), 'InPublication' => array(AmazonDynamoDB::TYPE_NUMBER => '0'), 'ProductCategory' => array(AmazonDynamoDB::TYPE_STRING => 'Book'))));
$dynamodb->batch($queue)->put_item(array('TableName' => 'ProductCatalog', 'Item' => array('Id' => array(AmazonDynamoDB::TYPE_NUMBER => '201'), 'Title' => array(AmazonDynamoDB::TYPE_STRING => '18-Bike-201'), 'Description' => array(AmazonDynamoDB::TYPE_STRING => '201 Description'), 'BicycleType' => array(AmazonDynamoDB::TYPE_STRING => 'Road'), 'Brand' => array(AmazonDynamoDB::TYPE_STRING => 'Mountain A'), 'Price' => array(AmazonDynamoDB::TYPE_NUMBER => '100'), 'Gender' => array(AmazonDynamoDB::TYPE_STRING => 'M'), 'Color' => array(AmazonDynamoDB::TYPE_ARRAY_OF_STRINGS => array('Red', 'Black')), 'ProductCategory' => array(AmazonDynamoDB::TYPE_STRING => 'Bicycle'))));
$dynamodb->batch($queue)->put_item(array('TableName' => 'ProductCatalog', 'Item' => array('Id' => array(AmazonDynamoDB::TYPE_NUMBER => '202'), 'Title' => array(AmazonDynamoDB::TYPE_STRING => '21-Bike-202'), 'Description' => array(AmazonDynamoDB::TYPE_STRING => '202 Description'), 'BicycleType' => array(AmazonDynamoDB::TYPE_STRING => 'Road'), 'Brand' => array(AmazonDynamoDB::TYPE_STRING => 'Brand-Company A'), 'Price' => array(AmazonDynamoDB::TYPE_NUMBER => '200'), 'Gender' => array(AmazonDynamoDB::TYPE_STRING => 'M'), 'Color' => array(AmazonDynamoDB::TYPE_ARRAY_OF_STRINGS => array('Green', 'Black')), 'ProductCategory' => array(AmazonDynamoDB::TYPE_STRING => 'Bicycle'))));
$dynamodb->batch($queue)->put_item(array('TableName' => 'ProductCatalog', 'Item' => array('Id' => array(AmazonDynamoDB::TYPE_NUMBER => '203'), 'Title' => array(AmazonDynamoDB::TYPE_STRING => '19-Bike-203'), 'Description' => array(AmazonDynamoDB::TYPE_STRING => '203 Description'), 'BicycleType' => array(AmazonDynamoDB::TYPE_STRING => 'Road'), 'Brand' => array(AmazonDynamoDB::TYPE_STRING => 'Brand-Company B'), 'Price' => array(AmazonDynamoDB::TYPE_NUMBER => '300'), 'Gender' => array(AmazonDynamoDB::TYPE_STRING => 'W'), 'Color' => array(AmazonDynamoDB::TYPE_ARRAY_OF_STRINGS => array('Red', 'Green', 'Black')), 'ProductCategory' => array(AmazonDynamoDB::TYPE_STRING => 'Bicycle'))));
$dynamodb->batch($queue)->put_item(array('TableName' => 'ProductCatalog', 'Item' => array('Id' => array(AmazonDynamoDB::TYPE_NUMBER => '204'), 'Title' => array(AmazonDynamoDB::TYPE_STRING => '18-Bike-204'), 'Description' => array(AmazonDynamoDB::TYPE_STRING => '204 Description'), 'BicycleType' => array(AmazonDynamoDB::TYPE_STRING => 'Mountain'), 'Brand' => array(AmazonDynamoDB::TYPE_STRING => 'Brand-Company B'), 'Price' => array(AmazonDynamoDB::TYPE_NUMBER => '400'), 'Gender' => array(AmazonDynamoDB::TYPE_STRING => 'W'), 'Color' => array(AmazonDynamoDB::TYPE_ARRAY_OF_STRINGS => array('Red')), 'ProductCategory' => array(AmazonDynamoDB::TYPE_STRING => 'Bicycle'))));
$dynamodb->batch($queue)->put_item(array('TableName' => 'ProductCatalog', 'Item' => array('Id' => array(AmazonDynamoDB::TYPE_NUMBER => '205'), 'Title' => array(AmazonDynamoDB::TYPE_STRING => '20-Bike-205'), 'Description' => array(AmazonDynamoDB::TYPE_STRING => '205 Description'), 'BicycleType' => array(AmazonDynamoDB::TYPE_STRING => 'Hybrid'), 'Brand' => array(AmazonDynamoDB::TYPE_STRING => 'Brand-Company C'), 'Price' => array(AmazonDynamoDB::TYPE_NUMBER => '500'), 'Gender' => array(AmazonDynamoDB::TYPE_STRING => 'B'), 'Color' => array(AmazonDynamoDB::TYPE_ARRAY_OF_STRINGS => array('Red', 'Black')), 'ProductCategory' => array(AmazonDynamoDB::TYPE_STRING => 'Bicycle'))));
$dynamodb->batch($queue)->put_item(array('TableName' => 'Forum', 'Item' => array('Name' => array(AmazonDynamoDB::TYPE_STRING => 'Amazon DynamoDB'), 'Category' => array(AmazonDynamoDB::TYPE_STRING => 'Amazon Web Services'), 'Threads' => array(AmazonDynamoDB::TYPE_NUMBER => '0'), 'Messages' => array(AmazonDynamoDB::TYPE_NUMBER => '0'), 'Views' => array(AmazonDynamoDB::TYPE_NUMBER => '1000'))));
$dynamodb->batch($queue)->put_item(array('TableName' => 'Forum', 'Item' => array('Name' => array(AmazonDynamoDB::TYPE_STRING => 'Amazon S3'), 'Category' => array(AmazonDynamoDB::TYPE_STRING => 'Amazon Web Services'), 'Threads' => array(AmazonDynamoDB::TYPE_NUMBER => '0'))));
$dynamodb->batch($queue)->put_item(array('TableName' => 'Reply', 'Item' => array('Id' => array(AmazonDynamoDB::TYPE_STRING => 'Amazon DynamoDB#DynamoDB Thread 1'), 'ReplyDateTime' => array(AmazonDynamoDB::TYPE_STRING => $fourteen_days_ago), 'Message' => array(AmazonDynamoDB::TYPE_STRING => 'DynamoDB Thread 1 Reply 2 text'), 'PostedBy' => array(AmazonDynamoDB::TYPE_STRING => 'User B'))));
public function run()
{
$this->_start_time = date('Y-m-d H:i:s');
$this->_cron_log_id = $this->Log->create_log($this->_start_time);
//need to get all products
$productCount = $this->Products->get_all_tracked_products_count();
$offset = 0;
//queue up products -- so we don't crash the machine with a lot of products
do {
$products = $this->Products->get_all_tracked_products();
$this->_products = array_merge($products, $this->_products);
$this->_numproducts += count($products);
$offset += 500;
} while ($offset < $productCount);
if (count($this->_crawlers) > 0) {
foreach ($this->_crawlers as $crawler) {
$crawlerName = strtolower(trim($crawler['name']));
$filePath = FCPATH . 'system/application/crawlers/' . $crawlerName . '.php';
//check to see if crawler exists
if (@file_exists($filePath) === false) {
throw new Exception('The crawler ' . $crawlerName . ' file does not exist. Please add it.');
}
require_once $filePath;
if (class_exists($crawlerName) === false) {
throw new Exception('The crawler ' . $crawlerName . ' crawler class is not available. Please check that its defined within the file');
}
$this->_currentCrawler = $crawlerName;
$this->_crawlerClass = new $crawlerName($crawler);
foreach ($this->_products as $product) {
if ($product === end($this->_products)) {
$this->_last_upc = $product->upc_code;
}
try {
$this->_current_product_id = $product->id;
$this->_crawlerClass->setIdentifier($product->upc_code);
$productData = $this->_crawlerClass->getProduct();
$offers = $this->_crawlerClass->getAllOffers();
$this->_numfoundproducts++;
foreach ($offers as $offer) {
$this->_numofferings++;
$crawled_products = array();
if (empty($offer['merchant'])) {
throw new Exception('Merchant not defined.');
}
$marketplace = $offer['marketplace'];
if (empty($marketplace)) {
throw new Exception('Marketplace not defined');
}
$r = preg_split('/(?=\\.[^.]+$)/', $marketplace);
$marketplace = strtolower($r[0]);
$merchant = $this->Crowl->getMerchantNameForKey($offer['merchant'], $product->upc_code, $marketplace, $offer['merchant'], $offer['merchant'], extractDomainByURL($offer['url']));
$this->_marketplaces[] = $offer['marketplace'];
if (empty($offer['price_floor']) || !is_numeric($offer['price_floor'])) {
throw new Exception('Price floor is undefined');
}
$crawled_products['ap'] = $offer['price_floor'];
$title = $productData->get("title");
if (empty($title)) {
throw new Exception('Title is undefined');
}
$crawled_products['t'] = $title;
$crawled_products['ar'] = $marketplace;
$crawled_products['il'] = null;
if (empty($offer['url'])) {
throw new Exception('Url is undefined');
}
$crawled_products['l'] = $offer['url'];
$crawled_products['mu'] = $offer['merchant_url'];
if (empty($offer['price']) || !is_numeric($offer['price'])) {
throw new Exception('Price is undefined');
}
$crawled_products['mpo'] = $offer['price'];
$hashKey = $marketplace . '#' . $product->upc_code;
$crawled_products['um'] = $hashKey;
$crawled_products['dt'] = time();
// Get the price from the last crawl
$dynamo = new AmazonDynamoDB();
$lastRecordResponse = $dynamo->query(array('TableName' => $this->_dynamo_products_trends, 'HashKeyValue' => array(AmazonDynamoDB::TYPE_STRING => $crawled_products['um']), 'ConsistentRead' => true, 'Limit' => 1, 'ScanIndexForward' => false));
var_dump($crawled_products);
$insert_response = $this->amzdb->insertData($this->_dynamo_products_trends, $crawled_products, $marketplace);
if (isset($insert_response->status) && $insert_response->status == 200) {
$this->Crowl->insertUPCMerchant($merchant, $product->upc_code, $marketplace, $crawled_products['mpo']);
}
$crowlMerchantName = $this->Crawl_data->crowlMerchantByMerchantName($merchant);
if (empty($crowlMerchantName)) {
log_message('error', __FILE__ . ' Crowl_m::amazon_lookup() Line ' . __LINE__ . ': crowl_merchant_name record not found for merchant ' . $offer['merchant']);
}
$crowlMerchantNameID = isset($crowlMerchantName->id) ? $crowlMerchantName->id : 0;
$violatedPrice = (double) $crawled_products['mpo'];
$dataVio = $this->Products->get_products_by_floor($product->upc_code, $violatedPrice, $product->store_id);
if ($dataVio) {
//$price_floor > $crowled_products['mpo'])
$violation['um'] = $hashKey;
$violation['dt'] = $crawled_products['dt'];
//time();
$violation['ss'] = date('Ymd', $violation['dt']) . '/' . md5($hashKey . $violation['dt']) . '.png';
$this->Crowl->updateViolationSummary($dataVio);
$this->Violator->updatePriceViolator($crowlMerchantNameID, $product->upc_code, 1, $crawled_products['dt']);
$lastCrawlPrice = (double) 0;
$hashKey = null;
$rangeKey = null;
if ($lastRecordResponse->isOK()) {
$mpo = isset($lastRecordResponse->body->Items->mpo->N) ? (double) $lastRecordResponse->body->Items->mpo->N : (double) 0;
// Merchant Price Offered
$lastCrawlPrice = $mpo;
$hashKey = isset($lastRecordResponse->body->Items->um->S) ? $lastRecordResponse->body->Items->um->S : null;
$rangeKey = isset($lastRecordResponse->body->Items->dt->N) ? $lastRecordResponse->body->Items->dt->N : null;
}
// Check if the price has changed
if ($lastCrawlPrice != $violatedPrice) {
if (!isset($uniquArr[$crawled_products['l']])) {
$uniquArr[$crawled_products['l']] = $violation['ss'];
$this->Crowl->addScreenShot($crawled_products['l'], $violation['ss'], false, $violatedPrice);
} else {
$violation['ss'] = $uniquArr[$crawled_products['l']];
}
} else {
$takeNewScreenShot = false;
// Get the violation screen shot
if (!is_null($hashKey) and !is_null($rangeKey)) {
$lastViolationResponse = $dynamo->query(array('TableName' => $this->_dynamo_violations, 'HashKeyValue' => array(AmazonDynamoDB::TYPE_STRING => (string) $hashKey), 'RangeKeyCondition' => array('ComparisonOperator' => AmazonDynamoDB::CONDITION_EQUAL, 'AttributeValueList' => array(array(AmazonDynamoDB::TYPE_NUMBER => (string) $rangeKey)))));
if ($lastViolationResponse->isOK() and $lastViolationResponse->body->Count == 1) {
if (isset($lastViolationResponse->body->Items->ss->S)) {
if (@fopen(get_instance()->config->item('s3_cname') . 'stickyvision/violations/' . $lastViolationResponse->body->Items->ss->S, 'r')) {
$violation['ss'] = $lastViolationResponse->body->Items->ss->S;
$takeNewScreenShot = false;
} else {
$takeNewScreenShot = true;
}
}
}
}
if ($takeNewScreenShot === true) {
$this->Crowl->addScreenShot($crawled_products['l'], $violation['ss'], false, $violatedPrice);
}
}
$this->amzdb->insertData($this->_dynamo_violations, $violation, $marketplace);
$violationFlag = true;
$violation = null;
} else {
// update price violators for products not in violation
$this->Violator->updatePriceViolator($crowlMerchantNameID, $product->upc_code, 0, $crawled_products['dt']);
}
$this->Crowl->updateUPCFlag($product->upc_code, $marketplace, '1');
}
//end offers iteration
$this->_crawlerClass->reset();
} catch (Exception $e) {
$exceptionDetail = array('crawler_log_id' => $this->_cron_log_id, 'crawler_name' => $this->_currentCrawler, 'message' => $e->getMessage(), 'created_at' => date('Y-m-d H:i:s'), 'product_id' => $this->_current_product_id);
echo 'exception detected..';
var_dump($exceptionDetail);
$this->CrawlerErrorLog->create_log($exceptionDetail);
//send an alert to the team
email_alertToTeam('Run Cron Job - Error Cron Log ID ' . $this->_cron_log_id . ' : ' . $e->getMessage());
}
}
}
}
$this->_end_time = date('Y-m-d H:i:s');
$this->Log->update_log($this->_cron_log_id, array('end_datetime' => $this->_end_time, 'products_count' => $this->_numfoundproducts, 'offerings_count' => $this->_numofferings));
//create cron_log -- this is needed to handle code that still thinks we need to use old crawler
$this->_marketplaces = array_unique($this->_marketplaces);
foreach ($this->_marketplaces as $marketplace) {
$domain = trim($marketplace);
$r = preg_split('/(?=\\.[^.]+$)/', $domain);
$marketplace = strtolower($r[0]);
$cron_log_id = $this->CronLog->create_log(array('datetime' => date('Y-m-d H:i:s'), 'key' => generate_rand(32), 'api_type' => $marketplace, 'start_datetime' => $this->_start_time, 'end_datetime' => $this->_end_time, 'google_count' => 0, 'last_UPC' => $this->_last_upc, 'run_from' => 'cronTab1'));
$this->Marketplace->add_retailer(strtolower($marketplace), strtolower($domain));
}
exit;
}
