/**
* Returns a singelton logger instance.
*
* @return Zend_Log
*/
public static function getInstance()
{
if (self::$_instance === null) {
$config = Zend_Registry::get('config');
self::$_instance = Zend_Log::factory(array(array('writerName' => 'Stream', 'writerParams' => array('stream' => 'php://output'), 'filterName' => 'Priority', 'filterParams' => array('priority' => constant('Zend_Log::' . $config->log->level)))));
}
return self::$_instance;
}
<?php
include_once '../bootstrap.php';
ini_set('memory_limit', '1024M');
set_time_limit(0);
// create logger
$logger = Yadda_Log::getInstance();
// get all feeds that have last_fetch = null or last_fetch more than 30m ago
$feedDb = Yadda_Db_Table::getInstance('feed');
$select = $feedDb->select()->setIntegrityCheck(false)->from('feed')->joinLeft('site', 'feed.site_id = site.id', array())->joinLeft('region', 'feed.region_id = region.id', array('region_lat' => 'lat', 'region_long' => 'long'))->where('feed.engine = ?', 'vuvuplaza')->order('feed.created');
$feeds = $feedDb->fetchAll($select);
foreach ($feeds as $feed) {
$logger->log('Processing feed for region "' . $feed->region_id . '" from site "' . $feed->site_id . '"', Zend_Log::DEBUG);
$logger->log('URL: ' . $feed->url, Zend_Log::DEBUG);
$logger->log('Last fetch: ' . $feed->last_fetch, Zend_Log::DEBUG);
try {
Yadda_Model_Feed::import($feed);
// mark the feed as fetched
$feedDb->update(array('last_fetch' => Yadda_Db::now(), 'modified' => Yadda_Db::now()), array('id = ?' => $feed->id));
} catch (Exception $e) {
$logger->log('Exception: ' . (string) $e, Zend_Log::ERR);
}
sleep(rand(0, 10));
}
/**
* Imports the deals from a feed.
*
* @param Zend_Db_Table_Row $feed
* @throws Yadda_Model_Exception
* @throws Yadda_Feed_Exception
* @return void
*/
public static function import($feed)
{
if (!isset(self::$engines[$feed->engine])) {
throw new Yadda_Model_Exception('Unknown feed type "' . $feed->engine . '"');
}
$class = 'Yadda_Feed_Engine_' . ucfirst(strtolower($feed->engine));
$engine = new $class();
$stubs = $engine->import($feed);
$logger = Yadda_Log::getInstance();
$dealDb = Yadda_Db_Table::getInstance('deal');
foreach ($stubs as $stub) {
ob_start();
print_r($stub);
$logger->log('Stub:' . "\n" . ob_get_clean(), Zend_Log::DEBUG);
// make sure we have the bare minimum
if ($stub->getGuid() === null || $stub->getTitle() === null) {
$logger->log('Rejected: Stub doesn\'t have GUID or title.', Zend_Log::DEBUG);
continue;
}
// do we already have this guid for this site?
$deal = $dealDb->fetchRow(array('site_id = ?' => $feed->site_id, 'guid = ?' => $stub->getGuid()));
if ($deal !== null) {
continue;
}
// do we already have an item with this title for this site today?
$deal = $dealDb->fetchRow(array('site_id = ?' => $feed->site_id, 'title = ?' => $stub->getTitle(), 'DATE_FORMAT(display_date, \'%Y%m%d\') = ?' => date('Ymd', strtotime($stub->getDate()))));
if ($deal !== null) {
continue;
}
$logger->log('New entry: ' . $stub->getGuid(), Zend_Log::INFO);
// determine geo coords
$geo = $stub->getGeo();
if ($geo === null && $feed->region_lat !== null && $feed->region_long !== null) {
// get deal title
$title = $stub->getTitle();
$title = utf8_decode($title);
$matches = array();
preg_match_all('#(at|from)\\s+((([A-Z][\\w\']+)\\s*)+)#', $title, $matches);
foreach ($matches[2] as $match) {
$logger->log('Testing "' . $match . '" for coords', Zend_Log::DEBUG);
$match = trim($match);
// remove prices/stopwords/etc.
if (preg_match('/^R\\d+$/', $match)) {
continue;
}
if ($match == 'On') {
continue;
}
// get coordinates
$test = Yadda_Model_Deal::getGeoFromPlaceName(array((double) $feed->region_lat, (double) $feed->region_long), $match);
if ($test !== null) {
$logger->log('Found geo coords: ' . join(' ', $test), Zend_Log::INFO);
$geo = $test;
break;
}
}
}
// insert the new deal
$dealDb->getAdapter()->beginTransaction();
try {
$insert = array('site_id' => $feed->site_id, 'region_id' => $feed->region_id, 'guid' => $stub->getGuid(), 'title' => $stub->getTitle(), 'description' => $stub->getDescription(), 'link' => $stub->getLink(), 'display_date' => date('Y-m-d H:i:s', strtotime($stub->getDate())), 'price' => $stub->getPrice(), 'value' => $stub->getValue(), 'discount' => $stub->getDiscount(), 'lat' => $geo !== null ? $geo[0] : null, 'long' => $geo !== null ? $geo[1] : null, 'status' => 'active', 'created' => Yadda_Db::now());
$dealId = $dealDb->insert($insert);
// check for images
foreach ($stub->getImages() as $image) {
$logger->log('New image: ' . $image, Zend_Log::INFO);
$imageId = $dealDb->getAdapter()->insert('image', array('deal_id' => $dealId, 'source' => $image, 'created' => Yadda_Db::now()));
}
$dealDb->getAdapter()->commit();
} catch (Zend_Exception $e) {
$dealDb->getAdapter()->rollBack();
throw new Yadda_Feed_Exception('Error importing deal: ' . $e->getMessage());
}
}
}
