function run_migrate($task, $args) { if (count($args) == 0) { throw new Exception('You must provide a app.'); } @(list($app, $env) = explode(':', $args[0])); if (!is_dir(sfConfig::get('sf_app_dir') . DIRECTORY_SEPARATOR . $app)) { throw new Exception('The app "' . $app . '" does not exist.'); } // define constants define('SF_ROOT_DIR', sfConfig::get('sf_root_dir')); define('SF_APP', $app); define('SF_ENVIRONMENT', $env ? $env : 'cli'); define('SF_DEBUG', true); // get configuration require_once SF_ROOT_DIR . DIRECTORY_SEPARATOR . 'apps' . DIRECTORY_SEPARATOR . SF_APP . DIRECTORY_SEPARATOR . 'config' . DIRECTORY_SEPARATOR . 'config.php'; $databaseManager = new sfDatabaseManager(); $databaseManager->initialize(); $migrator = new sfMigrator(); // if no other arguments besides app, then migrate to latest version if (count($args) == 1) { $runMigrationsCount = $migrator->migrate(); } elseif (isset($args[1]) && ctype_digit($args[1])) { $runMigrationsCount = $migrator->migrate($args[1]); } else { throw new Exception('You can provide a destination migration number as a second parameter'); } $currentVersion = $migrator->getCurrentVersion(); pake_echo_action('migrations', 'migrated ' . $runMigrationsCount . ' step(s)'); pake_echo_action('migrations', 'current database version: ' . $currentVersion); }
protected function execute($arguments = array(), $options = array()) { $databaseManager = new sfDatabaseManager($this->configuration); $databaseManager->initialize($this->configuration); $scraper = new GovernorScraper($options['test_mode'], $options['debug_mode'], $this->configuration); $scraper->execute(); }
protected function execute($arguments = array(), $options = array()) { $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true); $databaseManager = new sfDatabaseManager($configuration); $databaseManager->initialize($configuration); $db = Doctrine_Manager::connection(); $models = array('Entity', 'Relationship', 'LsList'); foreach ($models as $model) { $modelAlias = strtolower($model); $updateAlias = $model == 'LsList' ? 'ls_list' : $modelAlias; //get records to update $q = LsDoctrineQuery::create()->select('id')->from($model . ' ' . $modelAlias)->where($modelAlias . '.last_user_id IS NULL')->limit($options['limit'])->setHydrationMode(Doctrine::HYDRATE_NONE); if (!count($rows = $q->execute())) { //nothing to update, go to next model continue; } foreach ($rows as $row) { $id = $row[0]; //get last_user_id $result = LsDoctrineQuery::create()->select('m.user_id')->from('Modification m')->where('m.object_model = ? AND m.object_id = ?', array($model, $id))->orderBy('m.id DESC')->setHydrationMode(Doctrine::HYDRATE_NONE)->fetchOne(); if ($lastUserId = $result[0]) { $query = 'UPDATE ' . $updateAlias . ' SET last_user_id=? WHERE id=?'; //use PDO for speed $db->execute($query, array($lastUserId, $id)); } else { throw new Exception("Couldn't find last_user_id for " . $model . ' #' . $id); } } //only update records of one model at a time break; } //DONE LsCli::beep(); }
protected function execute($arguments = array(), $options = array()) { if (!$this->safeToRun()) { print "Process already running!\n"; die; } $timer = sfTimerManager::getTimer('execute'); $databaseManager = new sfDatabaseManager($this->configuration); $databaseManager->initialize($this->configuration); //set up index $index = EntityTable::getLuceneIndex(); //delete deleted entities $q = LsDoctrineQuery::create()->from('Entity e')->where('e.is_deleted = ?', true)->setHydrationMode(Doctrine::HYDRATE_ARRAY); foreach ($q->execute() as $entity) { if ($hits = $index->find('key:' . $entity['id'])) { if ($options['debug_mode']) { printf("Deleting index for Entity %s\n", $entity['id']); } foreach ($hits as $hit) { $index->delete($hit->id); } } } printf("Memory used: %s\n", LsNumber::makeBytesReadable(memory_get_usage())); printf("Index size: %s\n", $index->count()); $timer->addTime(); printf("Run time: %s\n", $timer->getElapsedTime()); sfTimerManager::clearTimers(); }
protected function execute($arguments = array(), $options = array()) { $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true); $databaseManager = new sfDatabaseManager($configuration); $databaseManager->initialize($configuration); $this->db = Doctrine_Manager::connection(); $this->s3 = new S3(sfConfig::get('app_amazon_access_key'), sfConfig::get('app_amazon_secret_key')); // hide strict errors error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE); $sql = "SELECT DISTINCT(r.source) FROM reference r ORDER BY id ASC LIMIT " . $options["limit"] . " OFFSET " . $options["offset"]; $stmt = $this->db->execute($sql); $sources = $stmt->fetchAll(PDO::FETCH_COLUMN); foreach ($sources as $source) { if (!$options["localdir"]) { $s3path = ReferenceTable::generateS3path($source); if (!$options["overwrite"] && $this->s3->getObjectInfo(sfConfig::get('app_amazon_s3_bucket'), $s3path) !== false) { print "ALREADY UPLOADED: " . $s3path . "\n"; continue; } } $this->writeTmpFile($source, $options["debug_mode"], $options["localdir"]); if (!$options["localdir"]) { $this->uploadTmpFile($source, $options["overwrite"], $options["debug_mode"]); if (unlink($this->getLocalPath($source, $options["local_dir"])) && $options["debug_mode"]) { print "REMOVED LOCAL: " . $source . " [" . sha1($source) . "]\n"; } } print "----------------------------------------------------------------\n"; } //DONE LsCli::beep(); }
protected function execute($arguments = array(), $options = array()) { $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true); $databaseManager = new sfDatabaseManager($configuration); $databaseManager->initialize($configuration); $db = Doctrine_Manager::connection(); //get person entities with all-caps names $sql = 'SELECT e.id, e.name FROM entity e ' . 'WHERE e.name <> \'\' AND e.primary_ext = ? AND CAST(UPPER(e.name) AS BINARY) = CAST(e.name AS BINARY)'; $stmt = $db->execute($sql, array('Person')); $names = $stmt->fetchAll(PDO::FETCH_ASSOC); foreach ($names as $ary) { $new = PersonTable::nameizePersonName($ary['name']); if ($new != $ary['name']) { $sql = 'UPDATE entity SET name = ? WHERE id = ?'; $stmt = $db->execute($sql, array($new, $ary['id'])); print "Changed Entity name " . $ary['name'] . " to " . $new . "\n"; } } //get aliases with all-caps names $sql = 'SELECT a.id, a.name FROM alias a LEFT JOIN entity e ON (e.id = a.entity_id) ' . 'WHERE a.name <> \'\' AND a.is_primary = 1 AND e.primary_ext = ? AND ' . 'CAST(UPPER(a.name) AS BINARY) = CAST(a.name AS BINARY)'; $stmt = $db->execute($sql, array('Person')); $names = $stmt->fetchAll(PDO::FETCH_ASSOC); foreach ($names as $ary) { $new = PersonTable::nameizePersonName($ary['name']); if ($new != $ary['name']) { $sql = 'UPDATE alias SET name = ? WHERE id = ?'; $stmt = $db->execute($sql, array($new, $ary['id'])); print "Changed Alias " . $ary['name'] . " to " . $new . "\n"; } } //DONE LsCli::beep(); }
protected function execute($arguments = array(), $options = array()) { $databaseManager = new sfDatabaseManager($this->configuration); $databaseManager->initialize($this->configuration); $scraper = new PublicCompanyScraper($options['test_mode'], $options['debug_mode'], $this->configuration, null, $browserTimeout = 60); if ($options['ticker']) { $scraper->setCompanyByTicker($options['ticker']); } else { if ($options['start_id']) { $scraper->setStartId($options['start_id']); } } $scraper->setSearchDepth($options['search_depth']); $years = explode(',', $options['years']); $scraper->setYears($years); $scraper->setLimit($options['limit']); $scraper->setSession($options['session']); if ($options['repeat_mode']) { $scraper->setRepeatMode(true); } if ($options['list_id']) { $scraper->setListId($options['list_id']); } $scraper->run(); }
protected function execute($arguments = array(), $options = array()) { $databaseManager = new sfDatabaseManager($this->configuration); $databaseManager->initialize($this->configuration); $db = Doctrine_Manager::connection(); $str = file_get_contents($options['file_name']); $lines = explode("\n", $str); foreach ($lines as $line) { $arr = explode(",", $line); $entity = Doctrine::getTable('Entity')->find($arr[0]); if (strlen($arr[1]) > 0) { $email = $entity->addEmail($arr[1]); if ($email) { if ($options['safe_mode'] == true) { $resp = $this->readline("Do name (" . $entity->name . ") and email (" . $arr[1] . ") match?"); } else { $resp = 'y'; } if ($resp == 'y') { $email->save(); } } else { echo "email failed to save\n"; } } else { echo "entity skipped\n"; } } }
protected function execute($arguments = array(), $options = array()) { $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true); $databaseManager = new sfDatabaseManager($configuration); $databaseManager->initialize($configuration); $fh = fopen($options['file_name'], 'a+'); $start = count(file($options['file_name'])); $this->db = Doctrine_Manager::connection(); $this->list = Doctrine::getTable('LsList')->find($options['list_id']); $q = $this->list->getListEntitiesByRankQuery(); $list_entities = $q->execute(); function func($value) { return $value['name']; } for ($i = $start; $i < count($list_entities); $i++) { $entity = $list_entities[$i]->Entity; $people = $entity->getRelatedEntitiesQuery(array('Person'), array(1, 2, 3, 4, 6, 7, 8, 9, 10))->execute(); $orgs = $entity->getRelatedEntitiesQuery(array('Org'), array(1, 2, 3, 4, 6, 7, 8, 9, 10))->execute(); $donations = $entity->getRelatedEntitiesQuery(array('Person'), array(5))->execute(); $people = implode("; ", array_map("func", $people->toArray())); $orgs = implode("; ", array_map("func", $orgs->toArray())); $donations = implode("; ", array_map("func", $donations->toArray())); $arr = array($entity, $people, $orgs, $donations); $str = implode("\t", $arr); fwrite($fh, $str . "\n"); } }
protected function execute($arguments = array(), $options = array()) { $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true); $databaseManager = new sfDatabaseManager($configuration); $databaseManager->initialize($configuration); $this->db = Doctrine_Manager::connection(); if ($options['house_senate'] == 'house') { $sql = 'select e1.id,e2.id from entity e1 left join political_candidate pc on pc.entity_id = e1.id left join political_candidate pc2 on pc2.house_fec_id = pc.house_fec_id left join entity e2 on e2.id = pc2.entity_id where e1.is_deleted = 0 and e2.is_deleted = 0 and e1.id <> e2.id and pc.id is not null and pc2.id is not null and pc.id <> pc2.id and pc.house_fec_id is not null and pc.house_fec_id <> "" and e1.id < e2.id group by e1.id,e2.id'; } else { if ($options['house_senate'] == 'senate') { $sql = 'select e1.id,e2.id from entity e1 left join political_candidate pc on pc.entity_id = e1.id left join political_candidate pc2 on pc2.senate_fec_id = pc.senate_fec_id left join entity e2 on e2.id = pc2.entity_id where e1.is_deleted = 0 and e2.is_deleted = 0 and e1.id <> e2.id and pc.id is not null and pc2.id is not null and pc.id <> pc2.id and pc.senate_fec_id is not null and pc.senate_fec_id <> "" and e1.id < e2.id group by e1.id,e2.id'; } else { echo 'House or Senate not selected...ending script' . "\n"; die; } } $stmt = $this->db->execute($sql); $rows = $stmt->fetchAll(); foreach ($rows as $row) { $e1 = Doctrine::getTable('Entity')->find($row[0]); $e2 = Doctrine::getTable('Entity')->find($row[1]); $mergedEntity = EntityTable::mergeAll($e1, $e2); $e2->setMerge(true); $e2->clearRelated(); $e2->delete(); echo ' Successfully merged ' . $e2->name . "\n"; if ($options['test_mode']) { die; } } }
function run_propel_build_sql_diff($task, $args) { if (!count($args)) { throw new Exception('You must provide the application.'); } $app = $args[0]; if (!is_dir(sfConfig::get('sf_app_dir') . DIRECTORY_SEPARATOR . $app)) { throw new Exception(sprintf('The app "%s" does not exist.', $app)); } run_propel_build_sql($task, $args); pake_echo_action('propel-sql-diff', "building database patch"); define('SF_ROOT_DIR', realpath(dirname(__FILE__) . '/../../../..')); define('SF_APP', $app); define('SF_ENVIRONMENT', isset($args[1]) ? $args[1] : 'dev'); define('SF_DEBUG', 1); require_once SF_ROOT_DIR . DIRECTORY_SEPARATOR . 'apps' . DIRECTORY_SEPARATOR . SF_APP . DIRECTORY_SEPARATOR . 'config' . DIRECTORY_SEPARATOR . 'config.php'; $databaseManager = new sfDatabaseManager(); $databaseManager->initialize(); $i = new dbInfo(); $i->loadFromDb(); $i2 = new dbInfo(); $i2->loadAllFilesInDir(sfConfig::get('sf_data_dir') . '/sql'); $diff = $i->getDiffWith($i2); $filename = sfConfig::get('sf_data_dir') . '/sql/diff.sql'; if ($diff == '') { pake_echo_comment("no difference found"); } pake_echo_action('propel-sql-diff', "writing file {$filename}"); file_put_contents($filename, $diff); }
protected function execute($arguments = array(), $options = array()) { $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true); $databaseManager = new sfDatabaseManager($configuration); $databaseManager->initialize($configuration); $this->debugMode = $options['debug_mode']; $this->db = Doctrine_Manager::connection(); $this->s3 = new S3(sfConfig::get('app_amazon_access_key'), sfConfig::get('app_amazon_secret_key')); // hide strict errors error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE); //get array of active entity image filenames if ($options['list_id']) { $sql = "SELECT i.id, i.filename, i.url, i.entity_id, e.name, e.primary_ext " . "FROM ls_list_entity le " . "LEFT JOIN image i ON (i.entity_id = le.entity_id) " . "LEFT JOIN entity e ON (e.id = le.entity_id) " . "WHERE le.list_id = ? AND le.is_deleted = 0 " . "AND i.is_deleted = 0 AND i.has_square = 0 " . "ORDER BY id DESC LIMIT " . $options['limit']; $params = array($options['list_id']); } else { $sql = "SELECT i.id, i.filename FROM image i " . "WHERE is_deleted = 0 AND has_square = 0 " . "ORDER BY id DESC LIMIT " . $options['limit']; $params = array(); } $stmt = $this->db->execute($sql, $params); $images = $stmt->fetchAll(PDO::FETCH_ASSOC); $count = count($images); foreach ($images as $image) { $this->printDebug("Processing image {$image['id']} belonging to entity {$image['entity_id']}..."); if ($this->downloadLarge($image['filename'])) { $this->printDebug("Downloaded large image from S3: " . $s3Path); } else { $s3Path = ImageTable::generateS3Url(ImageTable::getPath($image['filename'], 'large')); $this->printDebug("Couldn't download large image from S3: " . $s3Path); if ($image['url']) { if ($this->downloadToTmp($image['url'], $image['filename'])) { $this->printDebug("Downloaded original image: " . $image['url']); } else { $this->printDebug("Couldn't download original image: " . $image['url']); if ($this->downloadFromGoogle($image['name'])) { $this->printDebug("Downloaded new image of {$image['name']} from google"); } else { $count--; continue; } } } else { $count--; continue; } } if (!$this->createSquare($image['filename'], $options['size'])) { $this->printDebug("Coudln't create square image: {$image['filename']}"); $count--; continue; } if ($this->uploadFile($image['filename'], $options['check_first'], $options['debug_mode'])) { $this->recordSquare($image['id']); } $count--; print $count . " images remaining...\n"; } //DONE LsCli::beep(); }
protected function execute($arguments = array(), $options = array()) { // initialize database manager $databaseManager = new sfDatabaseManager($this->configuration); $databaseManager->initialize($this->configuration); $fortune_scraper = new Fortune1000Scraper($options['test_mode'], $options['debug_mode'], $this->configuration); $fortune_scraper->run(); }
protected function execute($arguments = array(), $options = array()) { $this->debugMode = $options['debug_mode']; $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true); $databaseManager = new sfDatabaseManager($configuration); $databaseManager->initialize($configuration); $db = Doctrine_Manager::connection(); $cache = LsApiCacheFilter::getApiCache(); $models = array('Entity', 'Relationship', 'LsList', 'Image', 'Alias', 'Reference'); $modelsToClear = array(); foreach ($models as $model) { //get max modification id from last execution if (!$this->hasMeta($model, 'last_modification_id')) { //if no max modification id set, then set it $sql = 'SELECT MAX(id) FROM modification m WHERE m.object_model = ?'; $stmt = $db->execute($sql, array($model)); $result = $stmt->fetch(PDO::FETCH_COLUMN); $this->saveMeta($model, 'last_modification_id', $result); } $lastId = $this->getMeta($model, 'last_modification_id'); //get records modified since then $sql = 'SELECT id, object_id FROM modification m WHERE id > ? AND object_model = ? AND is_create = 0 ORDER BY id'; $stmt = $db->execute($sql, array($lastId, $model)); $maxId = null; $rows = $stmt->fetchAll(PDO::FETCH_ASSOC); if (count($rows)) { $modelsToClear[] = $model; } foreach ($rows as $row) { $this->printDebug('Clearing API cache for ' . $model . ' ' . $row['object_id']); //clear record cache $cacheClearMethod = 'get' . $model . 'Patterns'; foreach (self::$cacheClearMethod($row['object_id']) as $pattern) { $cache->removePattern($pattern); } $maxId = $row['id']; } //set new max modification id if ($maxId) { $this->saveMeta($model, 'last_modification_id', $maxId); } } //we're not done yet! gotta clear searches! if (in_array('Entity', $modelsToClear)) { $this->printDebug('Clearing API cache for Entity search'); $cache->removePattern('/entities*'); $cache->removePattern('/batch/entities*'); } if (in_array('Relationship', $modelsToClear)) { $this->printDebug('Clearing API cache for Relationship search'); $cache->removePattern('/relationships*'); $cache->removePattern('/batch/relationships*'); } if (in_array('LsList', $modelsToClear)) { $this->printDebug('Clearing API cache for List search'); $cache->removePattern('/lists*'); } }
protected function execute($arguments = array(), $options = array()) { $databaseManager = new sfDatabaseManager($this->configuration); $databaseManager->initialize($this->configuration); $refs = Doctrine::getTable('Reference')->findBySource(''); foreach ($refs as $ref) { $ref->delete(); } }
protected function execute($arguments = array(), $options = array()) { $databaseManager = new sfDatabaseManager($this->configuration); $databaseManager->initialize($this->configuration); $entities = LsDoctrineQuery::create()->from('Entity e')->where('CHAR_LENGTH(e.name) = 100')->groupBy('e.name')->execute(); foreach ($entities as $e) { echo $e->id . ' :: ' . $e->name; echo "\n"; $duplicates = LsDoctrineQuery::create()->from('Entity e')->where('e.id != ? and e.name = ?', array($e->id, $e->name))->execute(); foreach ($duplicates as $e2) { $mergedEntity = EntityTable::mergeAll($e, $e2); $e2->setMerge(true); $e2->clearRelated(); $e2->delete(); echo ' Successfully merged ' . $e2->name . "\n"; } } /*$id = 0; $industry = LsDoctrineQuery::create() ->from('Industry i') ->where('id > ?',$id) ->fetchOne(); while ($industry) { $matches = LsDoctrineQuery::create() ->from('Industry i') ->where('i.name = ? and i.id != ?', array($industry->name,$industry->id)) ->execute(); echo $matches->count() . '...'; foreach($matches as $match) { $bis = LsDoctrineQuery::create() ->from('BusinessIndustry bi') ->where('bi.industry_id = ?', $match->id) ->execute(); foreach($bis as $b) { $b->industry_id = $industry->id; $b->save(); $match->delete(); } } $id = $industry->id; $industry = LsDoctrineQuery::create() ->from('Industry i') ->where('id > ?',$id) ->fetchOne(); } $images = LsQuery::getByModelAndFieldsQuery('Image',array('caption' => 'Array')); foreach($images as $image) { $image->caption = 'From the Biographical Directory of the United States Congress'; $image->save(); }*/ }
protected function execute($arguments = array(), $options = array()) { $databaseManager = new sfDatabaseManager($this->configuration); $databaseManager->initialize($this->configuration); $scraper = new ForbesPrivateCompaniesScraper($options['test_mode'], $options['debug_mode'], $this->configuration); $scraper->setLimit($options['limit']); $scraper->setYear($options['year']); $scraper->execute(); }
protected function execute($arguments = array(), $options = array()) { $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true); $databaseManager = new sfDatabaseManager($configuration); $databaseManager->initialize($configuration); $this->db = Doctrine_Manager::connection(); $this->s3 = new S3(sfConfig::get('app_amazon_access_key'), sfConfig::get('app_amazon_secret_key')); $types = explode(",", $options['types']); // hide strict errors error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE); if (count(array_intersect($types, array("profile", "small", "large")))) { //get array of active entity image filenames $sql = "SELECT i.filename FROM image i WHERE is_deleted = 0 ORDER BY id DESC LIMIT " . $options['limit']; $stmt = $this->db->execute($sql); $images = $stmt->fetchAll(PDO::FETCH_ASSOC); $count = count($images); foreach ($images as $image) { foreach ($types as $type) { if (in_array($type, array("system", "user"))) { continue; } $this->uploadFile($type, $image['filename'], $options['check_first'], $options['debug_mode']); } $count--; print $count . " entities remaining...\n"; } } if (in_array("system", $types)) { //do system images if ($handle = opendir(sfConfig::get('sf_image_dir') . "/system")) { /* This is the correct way to loop over the directory. */ while (false !== ($file = readdir($handle))) { if (strpos($file, ".") == 0) { continue; } $this->uploadFile("system", $file, $options['check_first'], $options['debug_mode']); } closedir($handle); } } if (in_array("user", $types)) { //get array of active entity image filenames $sql = "SELECT p.filename FROM sf_guard_user_profile p WHERE p.filename IS NOT NULL"; $stmt = $this->db->execute($sql); $files = $stmt->fetchAll(PDO::FETCH_COLUMN); $count = count($files); foreach ($files as $file) { foreach (array("profile", "small") as $type) { $this->uploadFile($type, $file, $options['check_first'], $options['debug_mode']); } $count--; print $count . " users remaining...\n"; } } //DONE LsCli::beep(); }
protected function execute($arguments = array(), $options = array()) { // initialize database manager $databaseManager = new sfDatabaseManager($this->configuration); $databaseManager->initialize($this->configuration); $proxy_scraper = new ProxyScraper($options['test_mode'], $options['debug_mode'], $this->configuration); $proxy_scraper->setCorpIds($options['limit'], $options['start_id'], $options['ticker']); $proxy_scraper->run(); }
protected function execute($arguments = array(), $options = array()) { // initialize database manager $databaseManager = new sfDatabaseManager($this->configuration); $databaseManager->initialize($this->configuration); $scraper = new OrgInfoScraper($options['test_mode'], $options['debug_mode'], $this->configuration); $scraper->setMetaName($options['meta_name']); $scraper->setOrgExtensions($options['org_extensions']); $scraper->run(); }
protected function execute($arguments = array(), $options = array()) { $databaseManager = new sfDatabaseManager($this->configuration); $databaseManager->initialize($this->configuration); $scraper = new FedSpendingScraper($options['test_mode'], $options['debug_mode'], $this->configuration); $scraper->setOrgLimit($options['org_limit']); $scraper->setRound($options['round']); $scraper->setFilingLimit($options['filing_limit']); $scraper->setYear($options['year']); $scraper->execute(); }
protected function execute($arguments = array(), $options = array()) { $databaseManager = new sfDatabaseManager($this->configuration); $databaseManager->initialize($this->configuration); $extensions1 = explode(',', $options['extensions1']); $extensions2 = explode(',', $options['extensions2']); $exact_first = $options['exact_first']; $start_id = $options['start_id']; $min_duplicates = $options['min_duplicates'] - 1; if (!in_array('Person', $extensions1) && !in_array('Person', $extensions2) && (!in_array('Org', $extensions1) && !in_array('Org', $extensions2))) { $this->printDebug('extension issues, exiting'); die; } $this->fileName = $options['file_name']; $entity = EntityTable::getByExtensionQuery($extensions1)->addWhere('e.id > ?', $start_id)->limit(1)->execute()->getFirst(); $ct = 0; $fh = fopen($this->fileName, 'a'); $been_done = array(); while ($entity) { $start_id = $entity->id; if (in_array('Person', $extensions1)) { $q = EntityTable::getByExtensionQuery($extensions2)->addWhere('person.name_last = ? and e.id <> ?', array($entity->name_last, $entity->id)); if (!$exact_first) { $q->addWhere('person.name_first like ?', substr($entity->name_first, 0, 1) . '%'); } else { $q->addWhere('person.name_first = ?', $entity->name_first); } if ($entity->name_middle) { $q->addWhere('person.name_middle like ? OR person.name_middle IS NULL', substr($entity->name_middle, 0, 1) . '%'); } } else { if (strlen($entity->name) > 8) { $q->addWhere('e.name like ?', '%' . $entity->name . '%'); } else { $q->addWhere('e.name = ?', $entity->name); } } if ($q->count() > $min_duplicates) { $matches = $q->execute(); $arr = array(); echo $entity->name . "\n"; foreach ($matches as $match) { $been_done[] = $match->id; $arr[] = $match->name; } $links = $entity->name . "\t"; $links .= implode("; ", $arr) . "\t"; $links .= 'http://littlesis.org/entity/view?id=' . $entity->id . "\n"; fwrite($fh, $links); } $entity = EntityTable::getByExtensionQuery($extensions1)->addWhere('e.id > ?', $start_id)->limit(1)->execute()->getFirst(); } fclose($fh); }
protected function execute($arguments = array(), $options = array()) { $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true); $databaseManager = new sfDatabaseManager($configuration); $databaseManager->initialize($configuration); $this->db = Doctrine_Manager::connection(); //get last processed merge modification if (!$this->hasMeta('log', 'last_modification_id')) { $lastModId = 0; } else { $lastModId = $this->getMeta('log', 'last_modification_id'); } //get merges since last processed merge modification $sql = 'SELECT m.* FROM modification m WHERE m.id > ? AND m.object_model = ? AND m.is_merge = 1 AND m.merge_object_id IS NOT NULL ORDER BY m.id DESC'; $stmt = $this->db->execute($sql, array($lastModId, 'Entity')); $mods = $stmt->fetchAll(PDO::FETCH_ASSOC); //for each merge, update any links involving the merged entity foreach ($mods as $mod) { if ($options['debug_mode']) { print "Updating links for merged entity " . $mod['object_id'] . " (to " . $mod['merge_object_id'] . ")\n"; } $sql = 'UPDATE link l SET entity1_id = ? WHERE entity1_id = ?'; $stmt = $this->db->execute($sql, array($mod['merge_object_id'], $mod['object_id'])); $sql = 'UPDATE link l SET entity2_id = ? WHERE entity2_id = ?'; $stmt = $this->db->execute($sql, array($mod['merge_object_id'], $mod['object_id'])); } //save highest merge modification for next time if (count($mods)) { $this->saveMeta('log', 'last_modification_id', $mods[0]['id']); } //get relationships with is_deleted = 0 and without links $sql = 'SELECT r.* FROM relationship r LEFT JOIN link l ON (r.id = l.relationship_id) WHERE r.is_deleted = 0 AND l.id IS NULL ORDER BY r.id LIMIT ' . $options['limit']; $stmt = $this->db->execute($sql); $rels = $stmt->fetchAll(PDO::FETCH_ASSOC); foreach ($rels as $rel) { if ($options['debug_mode']) { print "Creating Links for Relationship " . $rel['id'] . "\n"; } $this->createLinks($rel); } //get relationships with is_deleted = 1 and with links $sql = 'SELECT r.* FROM relationship r LEFT JOIN link l ON (r.id = l.relationship_id) WHERE r.is_deleted = 1 AND l.id IS NOT NULL GROUP BY r.id LIMIT ' . $options['limit']; $stmt = $this->db->execute($sql); $rels = $stmt->fetchAll(PDO::FETCH_ASSOC); foreach ($rels as $rel) { if ($options['debug_mode']) { print "Deleting Links for Relationship " . $rel['id'] . "\n"; } $this->deleteLinks($rel); } //DONE LsCli::beep(); }
protected function execute($arguments = array(), $options = array()) { $databaseManager = new sfDatabaseManager($this->configuration); $databaseManager->initialize($this->configuration); $scraper = new TickerScraper($options['test_mode'], $options['debug_mode'], $this->configuration, null, $browserTimeout = 60); if (!in_array($options['exchange'], array('nasdaq', 'amex', 'nyse'))) { $this->printDebug("you need to set exchange to nasdaq, amex, or nyse"); } $scraper->setMinMarketCap($options['min_market_cap']); $scraper->setExchange($options['exchange']); $scraper->run(); }
protected function execute($arguments = array(), $options = array()) { $configuration = ProjectConfiguration::getApplicationConfiguration($options['application'], $options['env'], true); $databaseManager = new sfDatabaseManager($configuration); $databaseManager->initialize($configuration); $q = EntityTable::getByExtensionQuery(array('Person', 'ElectedRepresentative'))->addWhere('summary like ? OR summary like ? OR summary like ? OR summary like ? OR summary like ? OR summary like ? OR summary like ? OR summary like ? OR summary like ?', array('(daughter%', '(son%', '(father%', '(mother%', '(cousin%', '(husband%', '(wife%', '(brother%', '(sister%'))->orderBy('person.name_last'); $members = $q->execute(); foreach ($members as $member) { if (preg_match('/\\([^\\)]*\\)/isu', $member->summary, $match)) { echo $member->name . ":\n"; if (preg_match_all('/(brother|sister|daughter|mother|father|wife|husband|cousin)\\sof\\s+([^\\;\\)\\,]*)(\\;|\\)|\\,)/isu', $match[0], $matches, PREG_SET_ORDER)) { foreach ($matches as $m) { echo "\t\t" . $m[1] . ' : of : ' . $m[2] . "\n"; $m[2] = str_replace('.', '', $m[2]); $parts = LsString::split($m[2]); $q = EntityTable::getByExtensionQuery(array('Person', 'ElectedRepresentative')); foreach ($parts as $part) { $q->addWhere('e.name like ?', '%' . $part . '%'); } $people = $q->execute(); $family = array(); foreach ($people as $person) { echo "\t\t\t\t" . $person->name . "\n"; if ($person->id != $member->id) { $family[] = $person; } } if (count($family) == 1) { $q = LsDoctrineQuery::create()->from('Relationship r')->where('(r.entity1_id = ? or r.entity2_id =?) and (r.entity1_id = ? or r.entity2_id = ?)', array($member->id, $member->id, $person->id, $person->id)); if (!$q->count()) { if ($description2 = FamilyTable::getDescription2($m[1], $family[0]->Gender->id)) { $relationship = new Relationship(); $relationship->setCategory('Family'); $relationship->Entity1 = $member; $relationship->Entity2 = $person; $relationship->description1 = $m[1]; $relationship->description2 = $description2; $relationship->save(); $ref = LsQuery::getByModelAndFieldsQuery('Reference', array('object_model' => 'Entity', 'object_id' => $member->id, 'name' => 'Congressional Biographical Directory'))->fetchOne(); if ($ref) { $relationship->addReference($ref->source, null, null, $ref->name, $ref->source_detail, $ref->publication_date); } echo "-------------------------------added relationship\n"; } } } } } echo "\n"; } } }
protected function execute($arguments = array(), $options = array()) { $configuration = ProjectConfiguration::getApplicationConfiguration($options['application'], $options['env'], true); $databaseManager = new sfDatabaseManager($configuration); $databaseManager->initialize($configuration); $dir = sfConfig::get('sf_data_dir') . DIRECTORY_SEPARATOR . 'fixtures'; $excludedModels = array('Modification', 'ModificationField'); Doctrine::loadModels(sfConfig::get('sf_lib_dir') . DIRECTORY_SEPARATOR . 'model' . DIRECTORY_SEPARATOR . 'doctrine'); $loadedModels = Doctrine::getLoadedModels(); $models = array_diff($loadedModels, $excludedModels); $data = new Doctrine_Data(); $data->exportData($dir, 'yml', $models, false); }
protected function execute($arguments = array(), $options = array()) { $databaseManager = new sfDatabaseManager($this->configuration); $databaseManager->initialize($this->configuration); $scraper = new LobbyingScraper($options['test_mode'], $options['debug_mode'], $this->configuration); $scraper->setMode($options['mode']); $scraper->setContinuous($options['continuous']); if ($options['filing_id']) { $scraper->setFilingId($options['filing_id']); } $scraper->setOrgIds($options['limit'], $options['start_id']); $scraper->run(); }
protected function init($arguments, $options) { //connect to DB $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true); $databaseManager = new sfDatabaseManager($configuration); $databaseManager->initialize($configuration); $this->db = Doctrine_Manager::connection(); //create insert statement $valStr = str_repeat('?, ', 5); $valStr = substr($valStr, 0, -2); $insertSql = 'INSERT INTO os_category (category_id, category_name, industry_id, industry_name, sector_name) VALUES (' . $valStr . ')'; $this->insertStmt = $this->db->prepare($insertSql); }
protected function init($arguments, $options) { $this->startTime = microtime(true); $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true); $databaseManager = new sfDatabaseManager($configuration); $databaseManager->initialize($configuration); $this->db = Doctrine_Manager::connection(); $rawDb = $databaseManager->getDatabase('raw'); $this->rawDb = Doctrine_Manager::connection($rawDb->getParameter('dsn')); $this->cycles = explode(',', $options['cycles']); $this->debugMode = $options['debug_mode']; $this->mode = $options['mode']; }
protected function execute($arguments = array(), $options = array()) { $databaseManager = new sfDatabaseManager($this->configuration); $databaseManager->initialize($this->configuration); $scraper = new FecContributionScraper($options['test_mode'], $options['debug_mode'], $this->configuration); $scraper->setLimit($options['limit']); $scraper->setYear($options['year']); $scraper->setEntityId($options['entity_id']); $scraper->setPrompt($options['prompt']); $scraper->setForce($options['force']); $scraper->setIgnoreMiddle($options['ignore_middle']); $scraper->execute(); }