<?php include_once '../bootstrap.php'; ini_set('memory_limit', '1024M'); set_time_limit(0); // create logger $logger = Yadda_Log::getInstance(); // get all feeds that have last_fetch = null or last_fetch more than 30m ago $feedDb = Yadda_Db_Table::getInstance('feed'); $select = $feedDb->select()->setIntegrityCheck(false)->from('feed')->joinLeft('site', 'feed.site_id = site.id', array())->joinLeft('region', 'feed.region_id = region.id', array('region_lat' => 'lat', 'region_long' => 'long'))->where('feed.engine = ?', 'vuvuplaza')->order('feed.created'); $feeds = $feedDb->fetchAll($select); foreach ($feeds as $feed) { $logger->log('Processing feed for region "' . $feed->region_id . '" from site "' . $feed->site_id . '"', Zend_Log::DEBUG); $logger->log('URL: ' . $feed->url, Zend_Log::DEBUG); $logger->log('Last fetch: ' . $feed->last_fetch, Zend_Log::DEBUG); try { Yadda_Model_Feed::import($feed); // mark the feed as fetched $feedDb->update(array('last_fetch' => Yadda_Db::now(), 'modified' => Yadda_Db::now()), array('id = ?' => $feed->id)); } catch (Exception $e) { $logger->log('Exception: ' . (string) $e, Zend_Log::ERR); } sleep(rand(0, 10)); }
/** * Imports the deals from a feed. * * @param Zend_Db_Table_Row $feed * @throws Yadda_Model_Exception * @throws Yadda_Feed_Exception * @return void */ public static function import($feed) { if (!isset(self::$engines[$feed->engine])) { throw new Yadda_Model_Exception('Unknown feed type "' . $feed->engine . '"'); } $class = 'Yadda_Feed_Engine_' . ucfirst(strtolower($feed->engine)); $engine = new $class(); $stubs = $engine->import($feed); $logger = Yadda_Log::getInstance(); $dealDb = Yadda_Db_Table::getInstance('deal'); foreach ($stubs as $stub) { ob_start(); print_r($stub); $logger->log('Stub:' . "\n" . ob_get_clean(), Zend_Log::DEBUG); // make sure we have the bare minimum if ($stub->getGuid() === null || $stub->getTitle() === null) { $logger->log('Rejected: Stub doesn\'t have GUID or title.', Zend_Log::DEBUG); continue; } // do we already have this guid for this site? $deal = $dealDb->fetchRow(array('site_id = ?' => $feed->site_id, 'guid = ?' => $stub->getGuid())); if ($deal !== null) { continue; } // do we already have an item with this title for this site today? $deal = $dealDb->fetchRow(array('site_id = ?' => $feed->site_id, 'title = ?' => $stub->getTitle(), 'DATE_FORMAT(display_date, \'%Y%m%d\') = ?' => date('Ymd', strtotime($stub->getDate())))); if ($deal !== null) { continue; } $logger->log('New entry: ' . $stub->getGuid(), Zend_Log::INFO); // determine geo coords $geo = $stub->getGeo(); if ($geo === null && $feed->region_lat !== null && $feed->region_long !== null) { // get deal title $title = $stub->getTitle(); $title = utf8_decode($title); $matches = array(); preg_match_all('#(at|from)\\s+((([A-Z][\\w\']+)\\s*)+)#', $title, $matches); foreach ($matches[2] as $match) { $logger->log('Testing "' . $match . '" for coords', Zend_Log::DEBUG); $match = trim($match); // remove prices/stopwords/etc. if (preg_match('/^R\\d+$/', $match)) { continue; } if ($match == 'On') { continue; } // get coordinates $test = Yadda_Model_Deal::getGeoFromPlaceName(array((double) $feed->region_lat, (double) $feed->region_long), $match); if ($test !== null) { $logger->log('Found geo coords: ' . join(' ', $test), Zend_Log::INFO); $geo = $test; break; } } } // insert the new deal $dealDb->getAdapter()->beginTransaction(); try { $insert = array('site_id' => $feed->site_id, 'region_id' => $feed->region_id, 'guid' => $stub->getGuid(), 'title' => $stub->getTitle(), 'description' => $stub->getDescription(), 'link' => $stub->getLink(), 'display_date' => date('Y-m-d H:i:s', strtotime($stub->getDate())), 'price' => $stub->getPrice(), 'value' => $stub->getValue(), 'discount' => $stub->getDiscount(), 'lat' => $geo !== null ? $geo[0] : null, 'long' => $geo !== null ? $geo[1] : null, 'status' => 'active', 'created' => Yadda_Db::now()); $dealId = $dealDb->insert($insert); // check for images foreach ($stub->getImages() as $image) { $logger->log('New image: ' . $image, Zend_Log::INFO); $imageId = $dealDb->getAdapter()->insert('image', array('deal_id' => $dealId, 'source' => $image, 'created' => Yadda_Db::now())); } $dealDb->getAdapter()->commit(); } catch (Zend_Exception $e) { $dealDb->getAdapter()->rollBack(); throw new Yadda_Feed_Exception('Error importing deal: ' . $e->getMessage()); } } }