function run_migrate($task, $args)
{
if (count($args) == 0) {
throw new Exception('You must provide a app.');
}
@(list($app, $env) = explode(':', $args[0]));
if (!is_dir(sfConfig::get('sf_app_dir') . DIRECTORY_SEPARATOR . $app)) {
throw new Exception('The app "' . $app . '" does not exist.');
}
// define constants
define('SF_ROOT_DIR', sfConfig::get('sf_root_dir'));
define('SF_APP', $app);
define('SF_ENVIRONMENT', $env ? $env : 'cli');
define('SF_DEBUG', true);
// get configuration
require_once SF_ROOT_DIR . DIRECTORY_SEPARATOR . 'apps' . DIRECTORY_SEPARATOR . SF_APP . DIRECTORY_SEPARATOR . 'config' . DIRECTORY_SEPARATOR . 'config.php';
$databaseManager = new sfDatabaseManager();
$databaseManager->initialize();
$migrator = new sfMigrator();
// if no other arguments besides app, then migrate to latest version
if (count($args) == 1) {
$runMigrationsCount = $migrator->migrate();
} elseif (isset($args[1]) && ctype_digit($args[1])) {
$runMigrationsCount = $migrator->migrate($args[1]);
} else {
throw new Exception('You can provide a destination migration number as a second parameter');
}
$currentVersion = $migrator->getCurrentVersion();
pake_echo_action('migrations', 'migrated ' . $runMigrationsCount . ' step(s)');
pake_echo_action('migrations', 'current database version: ' . $currentVersion);
}
protected function execute($arguments = array(), $options = array())
{
$databaseManager = new sfDatabaseManager($this->configuration);
$databaseManager->initialize($this->configuration);
$scraper = new GovernorScraper($options['test_mode'], $options['debug_mode'], $this->configuration);
$scraper->execute();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$db = Doctrine_Manager::connection();
$models = array('Entity', 'Relationship', 'LsList');
foreach ($models as $model) {
$modelAlias = strtolower($model);
$updateAlias = $model == 'LsList' ? 'ls_list' : $modelAlias;
//get records to update
$q = LsDoctrineQuery::create()->select('id')->from($model . ' ' . $modelAlias)->where($modelAlias . '.last_user_id IS NULL')->limit($options['limit'])->setHydrationMode(Doctrine::HYDRATE_NONE);
if (!count($rows = $q->execute())) {
//nothing to update, go to next model
continue;
}
foreach ($rows as $row) {
$id = $row[0];
//get last_user_id
$result = LsDoctrineQuery::create()->select('m.user_id')->from('Modification m')->where('m.object_model = ? AND m.object_id = ?', array($model, $id))->orderBy('m.id DESC')->setHydrationMode(Doctrine::HYDRATE_NONE)->fetchOne();
if ($lastUserId = $result[0]) {
$query = 'UPDATE ' . $updateAlias . ' SET last_user_id=? WHERE id=?';
//use PDO for speed
$db->execute($query, array($lastUserId, $id));
} else {
throw new Exception("Couldn't find last_user_id for " . $model . ' #' . $id);
}
}
//only update records of one model at a time
break;
}
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
if (!$this->safeToRun()) {
print "Process already running!\n";
die;
}
$timer = sfTimerManager::getTimer('execute');
$databaseManager = new sfDatabaseManager($this->configuration);
$databaseManager->initialize($this->configuration);
//set up index
$index = EntityTable::getLuceneIndex();
//delete deleted entities
$q = LsDoctrineQuery::create()->from('Entity e')->where('e.is_deleted = ?', true)->setHydrationMode(Doctrine::HYDRATE_ARRAY);
foreach ($q->execute() as $entity) {
if ($hits = $index->find('key:' . $entity['id'])) {
if ($options['debug_mode']) {
printf("Deleting index for Entity %s\n", $entity['id']);
}
foreach ($hits as $hit) {
$index->delete($hit->id);
}
}
}
printf("Memory used: %s\n", LsNumber::makeBytesReadable(memory_get_usage()));
printf("Index size: %s\n", $index->count());
$timer->addTime();
printf("Run time: %s\n", $timer->getElapsedTime());
sfTimerManager::clearTimers();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$this->db = Doctrine_Manager::connection();
$this->s3 = new S3(sfConfig::get('app_amazon_access_key'), sfConfig::get('app_amazon_secret_key'));
// hide strict errors
error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE);
$sql = "SELECT DISTINCT(r.source) FROM reference r ORDER BY id ASC LIMIT " . $options["limit"] . " OFFSET " . $options["offset"];
$stmt = $this->db->execute($sql);
$sources = $stmt->fetchAll(PDO::FETCH_COLUMN);
foreach ($sources as $source) {
if (!$options["localdir"]) {
$s3path = ReferenceTable::generateS3path($source);
if (!$options["overwrite"] && $this->s3->getObjectInfo(sfConfig::get('app_amazon_s3_bucket'), $s3path) !== false) {
print "ALREADY UPLOADED: " . $s3path . "\n";
continue;
}
}
$this->writeTmpFile($source, $options["debug_mode"], $options["localdir"]);
if (!$options["localdir"]) {
$this->uploadTmpFile($source, $options["overwrite"], $options["debug_mode"]);
if (unlink($this->getLocalPath($source, $options["local_dir"])) && $options["debug_mode"]) {
print "REMOVED LOCAL: " . $source . " [" . sha1($source) . "]\n";
}
}
print "----------------------------------------------------------------\n";
}
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$db = Doctrine_Manager::connection();
//get person entities with all-caps names
$sql = 'SELECT e.id, e.name FROM entity e ' . 'WHERE e.name <> \'\' AND e.primary_ext = ? AND CAST(UPPER(e.name) AS BINARY) = CAST(e.name AS BINARY)';
$stmt = $db->execute($sql, array('Person'));
$names = $stmt->fetchAll(PDO::FETCH_ASSOC);
foreach ($names as $ary) {
$new = PersonTable::nameizePersonName($ary['name']);
if ($new != $ary['name']) {
$sql = 'UPDATE entity SET name = ? WHERE id = ?';
$stmt = $db->execute($sql, array($new, $ary['id']));
print "Changed Entity name " . $ary['name'] . " to " . $new . "\n";
}
}
//get aliases with all-caps names
$sql = 'SELECT a.id, a.name FROM alias a LEFT JOIN entity e ON (e.id = a.entity_id) ' . 'WHERE a.name <> \'\' AND a.is_primary = 1 AND e.primary_ext = ? AND ' . 'CAST(UPPER(a.name) AS BINARY) = CAST(a.name AS BINARY)';
$stmt = $db->execute($sql, array('Person'));
$names = $stmt->fetchAll(PDO::FETCH_ASSOC);
foreach ($names as $ary) {
$new = PersonTable::nameizePersonName($ary['name']);
if ($new != $ary['name']) {
$sql = 'UPDATE alias SET name = ? WHERE id = ?';
$stmt = $db->execute($sql, array($new, $ary['id']));
print "Changed Alias " . $ary['name'] . " to " . $new . "\n";
}
}
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$databaseManager = new sfDatabaseManager($this->configuration);
$databaseManager->initialize($this->configuration);
$scraper = new PublicCompanyScraper($options['test_mode'], $options['debug_mode'], $this->configuration, null, $browserTimeout = 60);
if ($options['ticker']) {
$scraper->setCompanyByTicker($options['ticker']);
} else {
if ($options['start_id']) {
$scraper->setStartId($options['start_id']);
}
}
$scraper->setSearchDepth($options['search_depth']);
$years = explode(',', $options['years']);
$scraper->setYears($years);
$scraper->setLimit($options['limit']);
$scraper->setSession($options['session']);
if ($options['repeat_mode']) {
$scraper->setRepeatMode(true);
}
if ($options['list_id']) {
$scraper->setListId($options['list_id']);
}
$scraper->run();
}
protected function execute($arguments = array(), $options = array())
{
$databaseManager = new sfDatabaseManager($this->configuration);
$databaseManager->initialize($this->configuration);
$db = Doctrine_Manager::connection();
$str = file_get_contents($options['file_name']);
$lines = explode("\n", $str);
foreach ($lines as $line) {
$arr = explode(",", $line);
$entity = Doctrine::getTable('Entity')->find($arr[0]);
if (strlen($arr[1]) > 0) {
$email = $entity->addEmail($arr[1]);
if ($email) {
if ($options['safe_mode'] == true) {
$resp = $this->readline("Do name (" . $entity->name . ") and email (" . $arr[1] . ") match?");
} else {
$resp = 'y';
}
if ($resp == 'y') {
$email->save();
}
} else {
echo "email failed to save\n";
}
} else {
echo "entity skipped\n";
}
}
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$fh = fopen($options['file_name'], 'a+');
$start = count(file($options['file_name']));
$this->db = Doctrine_Manager::connection();
$this->list = Doctrine::getTable('LsList')->find($options['list_id']);
$q = $this->list->getListEntitiesByRankQuery();
$list_entities = $q->execute();
function func($value)
{
return $value['name'];
}
for ($i = $start; $i < count($list_entities); $i++) {
$entity = $list_entities[$i]->Entity;
$people = $entity->getRelatedEntitiesQuery(array('Person'), array(1, 2, 3, 4, 6, 7, 8, 9, 10))->execute();
$orgs = $entity->getRelatedEntitiesQuery(array('Org'), array(1, 2, 3, 4, 6, 7, 8, 9, 10))->execute();
$donations = $entity->getRelatedEntitiesQuery(array('Person'), array(5))->execute();
$people = implode("; ", array_map("func", $people->toArray()));
$orgs = implode("; ", array_map("func", $orgs->toArray()));
$donations = implode("; ", array_map("func", $donations->toArray()));
$arr = array($entity, $people, $orgs, $donations);
$str = implode("\t", $arr);
fwrite($fh, $str . "\n");
}
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$this->db = Doctrine_Manager::connection();
if ($options['house_senate'] == 'house') {
$sql = 'select e1.id,e2.id from entity e1 left join political_candidate pc on pc.entity_id = e1.id left join political_candidate pc2 on pc2.house_fec_id = pc.house_fec_id left join entity e2 on e2.id = pc2.entity_id where e1.is_deleted = 0 and e2.is_deleted = 0 and e1.id <> e2.id and pc.id is not null and pc2.id is not null and pc.id <> pc2.id and pc.house_fec_id is not null and pc.house_fec_id <> "" and e1.id < e2.id group by e1.id,e2.id';
} else {
if ($options['house_senate'] == 'senate') {
$sql = 'select e1.id,e2.id from entity e1 left join political_candidate pc on pc.entity_id = e1.id left join political_candidate pc2 on pc2.senate_fec_id = pc.senate_fec_id left join entity e2 on e2.id = pc2.entity_id where e1.is_deleted = 0 and e2.is_deleted = 0 and e1.id <> e2.id and pc.id is not null and pc2.id is not null and pc.id <> pc2.id and pc.senate_fec_id is not null and pc.senate_fec_id <> "" and e1.id < e2.id group by e1.id,e2.id';
} else {
echo 'House or Senate not selected...ending script' . "\n";
die;
}
}
$stmt = $this->db->execute($sql);
$rows = $stmt->fetchAll();
foreach ($rows as $row) {
$e1 = Doctrine::getTable('Entity')->find($row[0]);
$e2 = Doctrine::getTable('Entity')->find($row[1]);
$mergedEntity = EntityTable::mergeAll($e1, $e2);
$e2->setMerge(true);
$e2->clearRelated();
$e2->delete();
echo ' Successfully merged ' . $e2->name . "\n";
if ($options['test_mode']) {
die;
}
}
}
function run_propel_build_sql_diff($task, $args)
{
if (!count($args)) {
throw new Exception('You must provide the application.');
}
$app = $args[0];
if (!is_dir(sfConfig::get('sf_app_dir') . DIRECTORY_SEPARATOR . $app)) {
throw new Exception(sprintf('The app "%s" does not exist.', $app));
}
run_propel_build_sql($task, $args);
pake_echo_action('propel-sql-diff', "building database patch");
define('SF_ROOT_DIR', realpath(dirname(__FILE__) . '/../../../..'));
define('SF_APP', $app);
define('SF_ENVIRONMENT', isset($args[1]) ? $args[1] : 'dev');
define('SF_DEBUG', 1);
require_once SF_ROOT_DIR . DIRECTORY_SEPARATOR . 'apps' . DIRECTORY_SEPARATOR . SF_APP . DIRECTORY_SEPARATOR . 'config' . DIRECTORY_SEPARATOR . 'config.php';
$databaseManager = new sfDatabaseManager();
$databaseManager->initialize();
$i = new dbInfo();
$i->loadFromDb();
$i2 = new dbInfo();
$i2->loadAllFilesInDir(sfConfig::get('sf_data_dir') . '/sql');
$diff = $i->getDiffWith($i2);
$filename = sfConfig::get('sf_data_dir') . '/sql/diff.sql';
if ($diff == '') {
pake_echo_comment("no difference found");
}
pake_echo_action('propel-sql-diff', "writing file {$filename}");
file_put_contents($filename, $diff);
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$this->debugMode = $options['debug_mode'];
$this->db = Doctrine_Manager::connection();
$this->s3 = new S3(sfConfig::get('app_amazon_access_key'), sfConfig::get('app_amazon_secret_key'));
// hide strict errors
error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE);
//get array of active entity image filenames
if ($options['list_id']) {
$sql = "SELECT i.id, i.filename, i.url, i.entity_id, e.name, e.primary_ext " . "FROM ls_list_entity le " . "LEFT JOIN image i ON (i.entity_id = le.entity_id) " . "LEFT JOIN entity e ON (e.id = le.entity_id) " . "WHERE le.list_id = ? AND le.is_deleted = 0 " . "AND i.is_deleted = 0 AND i.has_square = 0 " . "ORDER BY id DESC LIMIT " . $options['limit'];
$params = array($options['list_id']);
} else {
$sql = "SELECT i.id, i.filename FROM image i " . "WHERE is_deleted = 0 AND has_square = 0 " . "ORDER BY id DESC LIMIT " . $options['limit'];
$params = array();
}
$stmt = $this->db->execute($sql, $params);
$images = $stmt->fetchAll(PDO::FETCH_ASSOC);
$count = count($images);
foreach ($images as $image) {
$this->printDebug("Processing image {$image['id']} belonging to entity {$image['entity_id']}...");
if ($this->downloadLarge($image['filename'])) {
$this->printDebug("Downloaded large image from S3: " . $s3Path);
} else {
$s3Path = ImageTable::generateS3Url(ImageTable::getPath($image['filename'], 'large'));
$this->printDebug("Couldn't download large image from S3: " . $s3Path);
if ($image['url']) {
if ($this->downloadToTmp($image['url'], $image['filename'])) {
$this->printDebug("Downloaded original image: " . $image['url']);
} else {
$this->printDebug("Couldn't download original image: " . $image['url']);
if ($this->downloadFromGoogle($image['name'])) {
$this->printDebug("Downloaded new image of {$image['name']} from google");
} else {
$count--;
continue;
}
}
} else {
$count--;
continue;
}
}
if (!$this->createSquare($image['filename'], $options['size'])) {
$this->printDebug("Coudln't create square image: {$image['filename']}");
$count--;
continue;
}
if ($this->uploadFile($image['filename'], $options['check_first'], $options['debug_mode'])) {
$this->recordSquare($image['id']);
}
$count--;
print $count . " images remaining...\n";
}
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
// initialize database manager
$databaseManager = new sfDatabaseManager($this->configuration);
$databaseManager->initialize($this->configuration);
$fortune_scraper = new Fortune1000Scraper($options['test_mode'], $options['debug_mode'], $this->configuration);
$fortune_scraper->run();
}
protected function execute($arguments = array(), $options = array())
{
$this->debugMode = $options['debug_mode'];
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$db = Doctrine_Manager::connection();
$cache = LsApiCacheFilter::getApiCache();
$models = array('Entity', 'Relationship', 'LsList', 'Image', 'Alias', 'Reference');
$modelsToClear = array();
foreach ($models as $model) {
//get max modification id from last execution
if (!$this->hasMeta($model, 'last_modification_id')) {
//if no max modification id set, then set it
$sql = 'SELECT MAX(id) FROM modification m WHERE m.object_model = ?';
$stmt = $db->execute($sql, array($model));
$result = $stmt->fetch(PDO::FETCH_COLUMN);
$this->saveMeta($model, 'last_modification_id', $result);
}
$lastId = $this->getMeta($model, 'last_modification_id');
//get records modified since then
$sql = 'SELECT id, object_id FROM modification m WHERE id > ? AND object_model = ? AND is_create = 0 ORDER BY id';
$stmt = $db->execute($sql, array($lastId, $model));
$maxId = null;
$rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
if (count($rows)) {
$modelsToClear[] = $model;
}
foreach ($rows as $row) {
$this->printDebug('Clearing API cache for ' . $model . ' ' . $row['object_id']);
//clear record cache
$cacheClearMethod = 'get' . $model . 'Patterns';
foreach (self::$cacheClearMethod($row['object_id']) as $pattern) {
$cache->removePattern($pattern);
}
$maxId = $row['id'];
}
//set new max modification id
if ($maxId) {
$this->saveMeta($model, 'last_modification_id', $maxId);
}
}
//we're not done yet! gotta clear searches!
if (in_array('Entity', $modelsToClear)) {
$this->printDebug('Clearing API cache for Entity search');
$cache->removePattern('/entities*');
$cache->removePattern('/batch/entities*');
}
if (in_array('Relationship', $modelsToClear)) {
$this->printDebug('Clearing API cache for Relationship search');
$cache->removePattern('/relationships*');
$cache->removePattern('/batch/relationships*');
}
if (in_array('LsList', $modelsToClear)) {
$this->printDebug('Clearing API cache for List search');
$cache->removePattern('/lists*');
}
}
protected function execute($arguments = array(), $options = array())
{
$databaseManager = new sfDatabaseManager($this->configuration);
$databaseManager->initialize($this->configuration);
$refs = Doctrine::getTable('Reference')->findBySource('');
foreach ($refs as $ref) {
$ref->delete();
}
}
protected function execute($arguments = array(), $options = array())
{
$databaseManager = new sfDatabaseManager($this->configuration);
$databaseManager->initialize($this->configuration);
$entities = LsDoctrineQuery::create()->from('Entity e')->where('CHAR_LENGTH(e.name) = 100')->groupBy('e.name')->execute();
foreach ($entities as $e) {
echo $e->id . ' :: ' . $e->name;
echo "\n";
$duplicates = LsDoctrineQuery::create()->from('Entity e')->where('e.id != ? and e.name = ?', array($e->id, $e->name))->execute();
foreach ($duplicates as $e2) {
$mergedEntity = EntityTable::mergeAll($e, $e2);
$e2->setMerge(true);
$e2->clearRelated();
$e2->delete();
echo ' Successfully merged ' . $e2->name . "\n";
}
}
/*$id = 0;
$industry = LsDoctrineQuery::create()
->from('Industry i')
->where('id > ?',$id)
->fetchOne();
while ($industry)
{
$matches = LsDoctrineQuery::create()
->from('Industry i')
->where('i.name = ? and i.id != ?', array($industry->name,$industry->id))
->execute();
echo $matches->count() . '...';
foreach($matches as $match)
{
$bis = LsDoctrineQuery::create()
->from('BusinessIndustry bi')
->where('bi.industry_id = ?', $match->id)
->execute();
foreach($bis as $b)
{
$b->industry_id = $industry->id;
$b->save();
$match->delete();
}
}
$id = $industry->id;
$industry = LsDoctrineQuery::create()
->from('Industry i')
->where('id > ?',$id)
->fetchOne();
}
$images = LsQuery::getByModelAndFieldsQuery('Image',array('caption' => 'Array'));
foreach($images as $image)
{
$image->caption = 'From the Biographical Directory of the United States Congress';
$image->save();
}*/
}
protected function execute($arguments = array(), $options = array())
{
$databaseManager = new sfDatabaseManager($this->configuration);
$databaseManager->initialize($this->configuration);
$scraper = new ForbesPrivateCompaniesScraper($options['test_mode'], $options['debug_mode'], $this->configuration);
$scraper->setLimit($options['limit']);
$scraper->setYear($options['year']);
$scraper->execute();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$this->db = Doctrine_Manager::connection();
$this->s3 = new S3(sfConfig::get('app_amazon_access_key'), sfConfig::get('app_amazon_secret_key'));
$types = explode(",", $options['types']);
// hide strict errors
error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE);
if (count(array_intersect($types, array("profile", "small", "large")))) {
//get array of active entity image filenames
$sql = "SELECT i.filename FROM image i WHERE is_deleted = 0 ORDER BY id DESC LIMIT " . $options['limit'];
$stmt = $this->db->execute($sql);
$images = $stmt->fetchAll(PDO::FETCH_ASSOC);
$count = count($images);
foreach ($images as $image) {
foreach ($types as $type) {
if (in_array($type, array("system", "user"))) {
continue;
}
$this->uploadFile($type, $image['filename'], $options['check_first'], $options['debug_mode']);
}
$count--;
print $count . " entities remaining...\n";
}
}
if (in_array("system", $types)) {
//do system images
if ($handle = opendir(sfConfig::get('sf_image_dir') . "/system")) {
/* This is the correct way to loop over the directory. */
while (false !== ($file = readdir($handle))) {
if (strpos($file, ".") == 0) {
continue;
}
$this->uploadFile("system", $file, $options['check_first'], $options['debug_mode']);
}
closedir($handle);
}
}
if (in_array("user", $types)) {
//get array of active entity image filenames
$sql = "SELECT p.filename FROM sf_guard_user_profile p WHERE p.filename IS NOT NULL";
$stmt = $this->db->execute($sql);
$files = $stmt->fetchAll(PDO::FETCH_COLUMN);
$count = count($files);
foreach ($files as $file) {
foreach (array("profile", "small") as $type) {
$this->uploadFile($type, $file, $options['check_first'], $options['debug_mode']);
}
$count--;
print $count . " users remaining...\n";
}
}
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
// initialize database manager
$databaseManager = new sfDatabaseManager($this->configuration);
$databaseManager->initialize($this->configuration);
$proxy_scraper = new ProxyScraper($options['test_mode'], $options['debug_mode'], $this->configuration);
$proxy_scraper->setCorpIds($options['limit'], $options['start_id'], $options['ticker']);
$proxy_scraper->run();
}
protected function execute($arguments = array(), $options = array())
{
// initialize database manager
$databaseManager = new sfDatabaseManager($this->configuration);
$databaseManager->initialize($this->configuration);
$scraper = new OrgInfoScraper($options['test_mode'], $options['debug_mode'], $this->configuration);
$scraper->setMetaName($options['meta_name']);
$scraper->setOrgExtensions($options['org_extensions']);
$scraper->run();
}
protected function execute($arguments = array(), $options = array())
{
$databaseManager = new sfDatabaseManager($this->configuration);
$databaseManager->initialize($this->configuration);
$scraper = new FedSpendingScraper($options['test_mode'], $options['debug_mode'], $this->configuration);
$scraper->setOrgLimit($options['org_limit']);
$scraper->setRound($options['round']);
$scraper->setFilingLimit($options['filing_limit']);
$scraper->setYear($options['year']);
$scraper->execute();
}
protected function execute($arguments = array(), $options = array())
{
$databaseManager = new sfDatabaseManager($this->configuration);
$databaseManager->initialize($this->configuration);
$extensions1 = explode(',', $options['extensions1']);
$extensions2 = explode(',', $options['extensions2']);
$exact_first = $options['exact_first'];
$start_id = $options['start_id'];
$min_duplicates = $options['min_duplicates'] - 1;
if (!in_array('Person', $extensions1) && !in_array('Person', $extensions2) && (!in_array('Org', $extensions1) && !in_array('Org', $extensions2))) {
$this->printDebug('extension issues, exiting');
die;
}
$this->fileName = $options['file_name'];
$entity = EntityTable::getByExtensionQuery($extensions1)->addWhere('e.id > ?', $start_id)->limit(1)->execute()->getFirst();
$ct = 0;
$fh = fopen($this->fileName, 'a');
$been_done = array();
while ($entity) {
$start_id = $entity->id;
if (in_array('Person', $extensions1)) {
$q = EntityTable::getByExtensionQuery($extensions2)->addWhere('person.name_last = ? and e.id <> ?', array($entity->name_last, $entity->id));
if (!$exact_first) {
$q->addWhere('person.name_first like ?', substr($entity->name_first, 0, 1) . '%');
} else {
$q->addWhere('person.name_first = ?', $entity->name_first);
}
if ($entity->name_middle) {
$q->addWhere('person.name_middle like ? OR person.name_middle IS NULL', substr($entity->name_middle, 0, 1) . '%');
}
} else {
if (strlen($entity->name) > 8) {
$q->addWhere('e.name like ?', '%' . $entity->name . '%');
} else {
$q->addWhere('e.name = ?', $entity->name);
}
}
if ($q->count() > $min_duplicates) {
$matches = $q->execute();
$arr = array();
echo $entity->name . "\n";
foreach ($matches as $match) {
$been_done[] = $match->id;
$arr[] = $match->name;
}
$links = $entity->name . "\t";
$links .= implode("; ", $arr) . "\t";
$links .= 'http://littlesis.org/entity/view?id=' . $entity->id . "\n";
fwrite($fh, $links);
}
$entity = EntityTable::getByExtensionQuery($extensions1)->addWhere('e.id > ?', $start_id)->limit(1)->execute()->getFirst();
}
fclose($fh);
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$this->db = Doctrine_Manager::connection();
//get last processed merge modification
if (!$this->hasMeta('log', 'last_modification_id')) {
$lastModId = 0;
} else {
$lastModId = $this->getMeta('log', 'last_modification_id');
}
//get merges since last processed merge modification
$sql = 'SELECT m.* FROM modification m WHERE m.id > ? AND m.object_model = ? AND m.is_merge = 1 AND m.merge_object_id IS NOT NULL ORDER BY m.id DESC';
$stmt = $this->db->execute($sql, array($lastModId, 'Entity'));
$mods = $stmt->fetchAll(PDO::FETCH_ASSOC);
//for each merge, update any links involving the merged entity
foreach ($mods as $mod) {
if ($options['debug_mode']) {
print "Updating links for merged entity " . $mod['object_id'] . " (to " . $mod['merge_object_id'] . ")\n";
}
$sql = 'UPDATE link l SET entity1_id = ? WHERE entity1_id = ?';
$stmt = $this->db->execute($sql, array($mod['merge_object_id'], $mod['object_id']));
$sql = 'UPDATE link l SET entity2_id = ? WHERE entity2_id = ?';
$stmt = $this->db->execute($sql, array($mod['merge_object_id'], $mod['object_id']));
}
//save highest merge modification for next time
if (count($mods)) {
$this->saveMeta('log', 'last_modification_id', $mods[0]['id']);
}
//get relationships with is_deleted = 0 and without links
$sql = 'SELECT r.* FROM relationship r LEFT JOIN link l ON (r.id = l.relationship_id) WHERE r.is_deleted = 0 AND l.id IS NULL ORDER BY r.id LIMIT ' . $options['limit'];
$stmt = $this->db->execute($sql);
$rels = $stmt->fetchAll(PDO::FETCH_ASSOC);
foreach ($rels as $rel) {
if ($options['debug_mode']) {
print "Creating Links for Relationship " . $rel['id'] . "\n";
}
$this->createLinks($rel);
}
//get relationships with is_deleted = 1 and with links
$sql = 'SELECT r.* FROM relationship r LEFT JOIN link l ON (r.id = l.relationship_id) WHERE r.is_deleted = 1 AND l.id IS NOT NULL GROUP BY r.id LIMIT ' . $options['limit'];
$stmt = $this->db->execute($sql);
$rels = $stmt->fetchAll(PDO::FETCH_ASSOC);
foreach ($rels as $rel) {
if ($options['debug_mode']) {
print "Deleting Links for Relationship " . $rel['id'] . "\n";
}
$this->deleteLinks($rel);
}
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$databaseManager = new sfDatabaseManager($this->configuration);
$databaseManager->initialize($this->configuration);
$scraper = new TickerScraper($options['test_mode'], $options['debug_mode'], $this->configuration, null, $browserTimeout = 60);
if (!in_array($options['exchange'], array('nasdaq', 'amex', 'nyse'))) {
$this->printDebug("you need to set exchange to nasdaq, amex, or nyse");
}
$scraper->setMinMarketCap($options['min_market_cap']);
$scraper->setExchange($options['exchange']);
$scraper->run();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($options['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$q = EntityTable::getByExtensionQuery(array('Person', 'ElectedRepresentative'))->addWhere('summary like ? OR summary like ? OR summary like ? OR summary like ? OR summary like ? OR summary like ? OR summary like ? OR summary like ? OR summary like ?', array('(daughter%', '(son%', '(father%', '(mother%', '(cousin%', '(husband%', '(wife%', '(brother%', '(sister%'))->orderBy('person.name_last');
$members = $q->execute();
foreach ($members as $member) {
if (preg_match('/\\([^\\)]*\\)/isu', $member->summary, $match)) {
echo $member->name . ":\n";
if (preg_match_all('/(brother|sister|daughter|mother|father|wife|husband|cousin)\\sof\\s+([^\\;\\)\\,]*)(\\;|\\)|\\,)/isu', $match[0], $matches, PREG_SET_ORDER)) {
foreach ($matches as $m) {
echo "\t\t" . $m[1] . ' : of : ' . $m[2] . "\n";
$m[2] = str_replace('.', '', $m[2]);
$parts = LsString::split($m[2]);
$q = EntityTable::getByExtensionQuery(array('Person', 'ElectedRepresentative'));
foreach ($parts as $part) {
$q->addWhere('e.name like ?', '%' . $part . '%');
}
$people = $q->execute();
$family = array();
foreach ($people as $person) {
echo "\t\t\t\t" . $person->name . "\n";
if ($person->id != $member->id) {
$family[] = $person;
}
}
if (count($family) == 1) {
$q = LsDoctrineQuery::create()->from('Relationship r')->where('(r.entity1_id = ? or r.entity2_id =?) and (r.entity1_id = ? or r.entity2_id = ?)', array($member->id, $member->id, $person->id, $person->id));
if (!$q->count()) {
if ($description2 = FamilyTable::getDescription2($m[1], $family[0]->Gender->id)) {
$relationship = new Relationship();
$relationship->setCategory('Family');
$relationship->Entity1 = $member;
$relationship->Entity2 = $person;
$relationship->description1 = $m[1];
$relationship->description2 = $description2;
$relationship->save();
$ref = LsQuery::getByModelAndFieldsQuery('Reference', array('object_model' => 'Entity', 'object_id' => $member->id, 'name' => 'Congressional Biographical Directory'))->fetchOne();
if ($ref) {
$relationship->addReference($ref->source, null, null, $ref->name, $ref->source_detail, $ref->publication_date);
}
echo "-------------------------------added relationship\n";
}
}
}
}
}
echo "\n";
}
}
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($options['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$dir = sfConfig::get('sf_data_dir') . DIRECTORY_SEPARATOR . 'fixtures';
$excludedModels = array('Modification', 'ModificationField');
Doctrine::loadModels(sfConfig::get('sf_lib_dir') . DIRECTORY_SEPARATOR . 'model' . DIRECTORY_SEPARATOR . 'doctrine');
$loadedModels = Doctrine::getLoadedModels();
$models = array_diff($loadedModels, $excludedModels);
$data = new Doctrine_Data();
$data->exportData($dir, 'yml', $models, false);
}
protected function execute($arguments = array(), $options = array())
{
$databaseManager = new sfDatabaseManager($this->configuration);
$databaseManager->initialize($this->configuration);
$scraper = new LobbyingScraper($options['test_mode'], $options['debug_mode'], $this->configuration);
$scraper->setMode($options['mode']);
$scraper->setContinuous($options['continuous']);
if ($options['filing_id']) {
$scraper->setFilingId($options['filing_id']);
}
$scraper->setOrgIds($options['limit'], $options['start_id']);
$scraper->run();
}
protected function init($arguments, $options)
{
//connect to DB
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$this->db = Doctrine_Manager::connection();
//create insert statement
$valStr = str_repeat('?, ', 5);
$valStr = substr($valStr, 0, -2);
$insertSql = 'INSERT INTO os_category (category_id, category_name, industry_id, industry_name, sector_name) VALUES (' . $valStr . ')';
$this->insertStmt = $this->db->prepare($insertSql);
}
protected function init($arguments, $options)
{
$this->startTime = microtime(true);
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$this->db = Doctrine_Manager::connection();
$rawDb = $databaseManager->getDatabase('raw');
$this->rawDb = Doctrine_Manager::connection($rawDb->getParameter('dsn'));
$this->cycles = explode(',', $options['cycles']);
$this->debugMode = $options['debug_mode'];
$this->mode = $options['mode'];
}
protected function execute($arguments = array(), $options = array())
{
$databaseManager = new sfDatabaseManager($this->configuration);
$databaseManager->initialize($this->configuration);
$scraper = new FecContributionScraper($options['test_mode'], $options['debug_mode'], $this->configuration);
$scraper->setLimit($options['limit']);
$scraper->setYear($options['year']);
$scraper->setEntityId($options['entity_id']);
$scraper->setPrompt($options['prompt']);
$scraper->setForce($options['force']);
$scraper->setIgnoreMiddle($options['ignore_middle']);
$scraper->execute();
}
