protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$this->db = Doctrine_Manager::connection();
$this->s3 = new S3(sfConfig::get('app_amazon_access_key'), sfConfig::get('app_amazon_secret_key'));
// hide strict errors
error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE);
$sql = "SELECT DISTINCT(r.source) FROM reference r ORDER BY id ASC LIMIT " . $options["limit"] . " OFFSET " . $options["offset"];
$stmt = $this->db->execute($sql);
$sources = $stmt->fetchAll(PDO::FETCH_COLUMN);
foreach ($sources as $source) {
if (!$options["localdir"]) {
$s3path = ReferenceTable::generateS3path($source);
if (!$options["overwrite"] && $this->s3->getObjectInfo(sfConfig::get('app_amazon_s3_bucket'), $s3path) !== false) {
print "ALREADY UPLOADED: " . $s3path . "\n";
continue;
}
}
$this->writeTmpFile($source, $options["debug_mode"], $options["localdir"]);
if (!$options["localdir"]) {
$this->uploadTmpFile($source, $options["overwrite"], $options["debug_mode"]);
if (unlink($this->getLocalPath($source, $options["local_dir"])) && $options["debug_mode"]) {
print "REMOVED LOCAL: " . $source . " [" . sha1($source) . "]\n";
}
}
print "----------------------------------------------------------------\n";
}
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
//set start time
$time = microtime(true);
//connect to raw database
$this->init($arguments, $options);
$count = 0;
if ($options['debug_mode']) {
print "Processing OpenSecrets lobbying agency records...\n";
}
//open file
$file = fopen(sfConfig::get('sf_root_dir') . '/data/opensecrets/lob_agency.txt', 'r');
//process rows
while ($data = fgetcsv($file, 1000, ',', '|')) {
$this->insertData($data);
$count++;
if ($options['debug_mode']) {
print "Processed OpenSecrets lobbying agency for lobbying record " . $data[0] . ": " . $data[2] . "\n";
flush();
}
}
$duration = microtime(true) - $time;
print $count . " OpenSecrets lobbying agency records processed in " . $duration . " s\n";
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$db = Doctrine_Manager::connection();
$models = array('Entity', 'Relationship', 'LsList');
foreach ($models as $model) {
$modelAlias = strtolower($model);
$updateAlias = $model == 'LsList' ? 'ls_list' : $modelAlias;
//get records to update
$q = LsDoctrineQuery::create()->select('id')->from($model . ' ' . $modelAlias)->where($modelAlias . '.last_user_id IS NULL')->limit($options['limit'])->setHydrationMode(Doctrine::HYDRATE_NONE);
if (!count($rows = $q->execute())) {
//nothing to update, go to next model
continue;
}
foreach ($rows as $row) {
$id = $row[0];
//get last_user_id
$result = LsDoctrineQuery::create()->select('m.user_id')->from('Modification m')->where('m.object_model = ? AND m.object_id = ?', array($model, $id))->orderBy('m.id DESC')->setHydrationMode(Doctrine::HYDRATE_NONE)->fetchOne();
if ($lastUserId = $result[0]) {
$query = 'UPDATE ' . $updateAlias . ' SET last_user_id=? WHERE id=?';
//use PDO for speed
$db->execute($query, array($lastUserId, $id));
} else {
throw new Exception("Couldn't find last_user_id for " . $model . ' #' . $id);
}
}
//only update records of one model at a time
break;
}
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$db = Doctrine_Manager::connection();
//get person entities with all-caps names
$sql = 'SELECT e.id, e.name FROM entity e ' . 'WHERE e.name <> \'\' AND e.primary_ext = ? AND CAST(UPPER(e.name) AS BINARY) = CAST(e.name AS BINARY)';
$stmt = $db->execute($sql, array('Person'));
$names = $stmt->fetchAll(PDO::FETCH_ASSOC);
foreach ($names as $ary) {
$new = PersonTable::nameizePersonName($ary['name']);
if ($new != $ary['name']) {
$sql = 'UPDATE entity SET name = ? WHERE id = ?';
$stmt = $db->execute($sql, array($new, $ary['id']));
print "Changed Entity name " . $ary['name'] . " to " . $new . "\n";
}
}
//get aliases with all-caps names
$sql = 'SELECT a.id, a.name FROM alias a LEFT JOIN entity e ON (e.id = a.entity_id) ' . 'WHERE a.name <> \'\' AND a.is_primary = 1 AND e.primary_ext = ? AND ' . 'CAST(UPPER(a.name) AS BINARY) = CAST(a.name AS BINARY)';
$stmt = $db->execute($sql, array('Person'));
$names = $stmt->fetchAll(PDO::FETCH_ASSOC);
foreach ($names as $ary) {
$new = PersonTable::nameizePersonName($ary['name']);
if ($new != $ary['name']) {
$sql = 'UPDATE alias SET name = ? WHERE id = ?';
$stmt = $db->execute($sql, array($new, $ary['id']));
print "Changed Alias " . $ary['name'] . " to " . $new . "\n";
}
}
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$this->debugMode = $options['debug_mode'];
$this->db = Doctrine_Manager::connection();
$this->s3 = new S3(sfConfig::get('app_amazon_access_key'), sfConfig::get('app_amazon_secret_key'));
// hide strict errors
error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE);
//get array of active entity image filenames
if ($options['list_id']) {
$sql = "SELECT i.id, i.filename, i.url, i.entity_id, e.name, e.primary_ext " . "FROM ls_list_entity le " . "LEFT JOIN image i ON (i.entity_id = le.entity_id) " . "LEFT JOIN entity e ON (e.id = le.entity_id) " . "WHERE le.list_id = ? AND le.is_deleted = 0 " . "AND i.is_deleted = 0 AND i.has_square = 0 " . "ORDER BY id DESC LIMIT " . $options['limit'];
$params = array($options['list_id']);
} else {
$sql = "SELECT i.id, i.filename FROM image i " . "WHERE is_deleted = 0 AND has_square = 0 " . "ORDER BY id DESC LIMIT " . $options['limit'];
$params = array();
}
$stmt = $this->db->execute($sql, $params);
$images = $stmt->fetchAll(PDO::FETCH_ASSOC);
$count = count($images);
foreach ($images as $image) {
$this->printDebug("Processing image {$image['id']} belonging to entity {$image['entity_id']}...");
if ($this->downloadLarge($image['filename'])) {
$this->printDebug("Downloaded large image from S3: " . $s3Path);
} else {
$s3Path = ImageTable::generateS3Url(ImageTable::getPath($image['filename'], 'large'));
$this->printDebug("Couldn't download large image from S3: " . $s3Path);
if ($image['url']) {
if ($this->downloadToTmp($image['url'], $image['filename'])) {
$this->printDebug("Downloaded original image: " . $image['url']);
} else {
$this->printDebug("Couldn't download original image: " . $image['url']);
if ($this->downloadFromGoogle($image['name'])) {
$this->printDebug("Downloaded new image of {$image['name']} from google");
} else {
$count--;
continue;
}
}
} else {
$count--;
continue;
}
}
if (!$this->createSquare($image['filename'], $options['size'])) {
$this->printDebug("Coudln't create square image: {$image['filename']}");
$count--;
continue;
}
if ($this->uploadFile($image['filename'], $options['check_first'], $options['debug_mode'])) {
$this->recordSquare($image['id']);
}
$count--;
print $count . " images remaining...\n";
}
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$this->db = Doctrine_Manager::connection();
$this->s3 = new S3(sfConfig::get('app_amazon_access_key'), sfConfig::get('app_amazon_secret_key'));
$types = explode(",", $options['types']);
// hide strict errors
error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE);
if (count(array_intersect($types, array("profile", "small", "large")))) {
//get array of active entity image filenames
$sql = "SELECT i.filename FROM image i WHERE is_deleted = 0 ORDER BY id DESC LIMIT " . $options['limit'];
$stmt = $this->db->execute($sql);
$images = $stmt->fetchAll(PDO::FETCH_ASSOC);
$count = count($images);
foreach ($images as $image) {
foreach ($types as $type) {
if (in_array($type, array("system", "user"))) {
continue;
}
$this->uploadFile($type, $image['filename'], $options['check_first'], $options['debug_mode']);
}
$count--;
print $count . " entities remaining...\n";
}
}
if (in_array("system", $types)) {
//do system images
if ($handle = opendir(sfConfig::get('sf_image_dir') . "/system")) {
/* This is the correct way to loop over the directory. */
while (false !== ($file = readdir($handle))) {
if (strpos($file, ".") == 0) {
continue;
}
$this->uploadFile("system", $file, $options['check_first'], $options['debug_mode']);
}
closedir($handle);
}
}
if (in_array("user", $types)) {
//get array of active entity image filenames
$sql = "SELECT p.filename FROM sf_guard_user_profile p WHERE p.filename IS NOT NULL";
$stmt = $this->db->execute($sql);
$files = $stmt->fetchAll(PDO::FETCH_COLUMN);
$count = count($files);
foreach ($files as $file) {
foreach (array("profile", "small") as $type) {
$this->uploadFile($type, $file, $options['check_first'], $options['debug_mode']);
}
$count--;
print $count . " users remaining...\n";
}
}
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$this->db = Doctrine_Manager::connection();
//get last processed merge modification
if (!$this->hasMeta('log', 'last_modification_id')) {
$lastModId = 0;
} else {
$lastModId = $this->getMeta('log', 'last_modification_id');
}
//get merges since last processed merge modification
$sql = 'SELECT m.* FROM modification m WHERE m.id > ? AND m.object_model = ? AND m.is_merge = 1 AND m.merge_object_id IS NOT NULL ORDER BY m.id DESC';
$stmt = $this->db->execute($sql, array($lastModId, 'Entity'));
$mods = $stmt->fetchAll(PDO::FETCH_ASSOC);
//for each merge, update any links involving the merged entity
foreach ($mods as $mod) {
if ($options['debug_mode']) {
print "Updating links for merged entity " . $mod['object_id'] . " (to " . $mod['merge_object_id'] . ")\n";
}
$sql = 'UPDATE link l SET entity1_id = ? WHERE entity1_id = ?';
$stmt = $this->db->execute($sql, array($mod['merge_object_id'], $mod['object_id']));
$sql = 'UPDATE link l SET entity2_id = ? WHERE entity2_id = ?';
$stmt = $this->db->execute($sql, array($mod['merge_object_id'], $mod['object_id']));
}
//save highest merge modification for next time
if (count($mods)) {
$this->saveMeta('log', 'last_modification_id', $mods[0]['id']);
}
//get relationships with is_deleted = 0 and without links
$sql = 'SELECT r.* FROM relationship r LEFT JOIN link l ON (r.id = l.relationship_id) WHERE r.is_deleted = 0 AND l.id IS NULL ORDER BY r.id LIMIT ' . $options['limit'];
$stmt = $this->db->execute($sql);
$rels = $stmt->fetchAll(PDO::FETCH_ASSOC);
foreach ($rels as $rel) {
if ($options['debug_mode']) {
print "Creating Links for Relationship " . $rel['id'] . "\n";
}
$this->createLinks($rel);
}
//get relationships with is_deleted = 1 and with links
$sql = 'SELECT r.* FROM relationship r LEFT JOIN link l ON (r.id = l.relationship_id) WHERE r.is_deleted = 1 AND l.id IS NOT NULL GROUP BY r.id LIMIT ' . $options['limit'];
$stmt = $this->db->execute($sql);
$rels = $stmt->fetchAll(PDO::FETCH_ASSOC);
foreach ($rels as $rel) {
if ($options['debug_mode']) {
print "Deleting Links for Relationship " . $rel['id'] . "\n";
}
$this->deleteLinks($rel);
}
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
//set start time
$time = microtime(true);
//connect to raw database
$this->init($arguments, $options);
$count = 0;
foreach ($this->cycles as $cycle) {
if ($options['debug_mode']) {
print "Processing OpenSecrets candidates from " . $cycle . " cycle...\n";
}
//open file
$file = fopen(sfConfig::get('sf_root_dir') . '/data/opensecrets/cands' . substr($cycle, -2) . '.txt', 'r');
//process rows
while ($data = fgetcsv($file, 1000, ",", "|")) {
if ($count >= $this->limit) {
if ($options['debug_mode']) {
print "*** Import limit reached; exiting...\n";
}
break;
}
if ($this->isCandidateDuplicate($data[1], $data[2], $data[0])) {
if ($options['debug_mode']) {
print "- Candidate " . $data[2] . " already exists; skipping...\n";
}
continue;
}
if (count($data) != 12) {
if ($options['debug_mode']) {
print "* Candidate " . $data[2] . " has " . count($data) . " fields; adjusting...\n";
}
$data = array_slice($data, 0, 2);
for ($n = 0; $n < 10; $n++) {
$data[] = null;
}
}
$data = $this->parseCandidateName($data);
$this->insertData($data);
$count++;
if ($options['debug_mode']) {
print "Processed OpenSecrets " . $cycle . " candidate record " . $data[2] . "\n";
flush();
}
}
}
$duration = microtime(true) - $time;
print $count . " OpenSecrets candidate records processed in " . $duration . " s\n";
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
//set start time
$time = microtime(true);
//connect to raw database
$this->init($arguments, $options);
$count = 0;
foreach ($this->cycles as $cycle) {
if ($options['debug_mode']) {
print "Processing OpenSecrets committees from " . $cycle . " cycle...\n";
}
//open file
$file = fopen(sfConfig::get('sf_root_dir') . '/data/opensecrets/cmtes' . substr($cycle, -2) . '.txt', 'r');
//process rows
while ($data = fgetcsv($file, 1000, ",", "|")) {
if ($count >= $this->limit) {
if ($options['debug_mode']) {
print "*** Import limit reached; exiting...\n";
}
break;
}
if ($this->isCommitteeDuplicate($data[0], $data[1])) {
if ($options['debug_mode']) {
print "- Committee " . $data[1] . " already exists for " . $data[0] . " cycle; skipping...\n";
}
continue;
}
if (count($data) != 14) {
$data = array_slice($data, 0, 3);
for ($n = 0; $n < 11; $n++) {
$data[] = null;
}
}
$this->insertData($data);
$count++;
if ($options['debug_mode']) {
print "Processed OpenSecrets " . $cycle . " committee record " . $data[1] . "\n";
flush();
}
}
}
$duration = microtime(true) - $time;
print $count . " OpenSecrets committee records processed in " . $duration . " s\n";
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
//set start time
$time = microtime(true);
//connect to raw database
$this->init($arguments, $options);
//determine how many rows alrady processed
$sql = 'SELECT COUNT(*) FROM os_pac_donation WHERE cycle = ?';
$stmt = $this->db->execute($sql, array($options['cycle']));
$count = $stmt->fetch(PDO::FETCH_COLUMN);
//open file
$file = fopen(sfConfig::get('sf_root_dir') . '/data/opensecrets/pacs' . substr($options['cycle'], -2) . '.csv', 'r');
//skip to last line in file
$line = 0;
while ($line < $count) {
fgetcsv($file, 1000);
$line++;
}
//process remaining rows
$limit = $line + $options['limit'];
while ($line < $limit && ($data = fgetcsv($file, 1000))) {
if (count($data) != 10) {
die('Unepexted number of columns on line ' . $line . ': ' . count($data));
}
//convert date format
if ($data[5]) {
$data[5] = date('Y-m-d', strtotime($data[8]));
}
$this->insertData($data);
$line++;
if ($options['debug_mode']) {
print "Processed OpenSecrets " . $options['cycle'] . " PAC donation record " . $data[1] . "\n";
flush();
}
}
$duration = microtime(true) - $time;
print $limit . " total OpenSecrets " . $options['cycle'] . " PAC donation records processed in " . $duration . " s\n";
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$db = Doctrine_Manager::connection();
try {
$db->beginTransaction();
$q = LsDoctrineQuery::create()->from('Entity e')->where('e.primary_ext IS NULL')->andWhere('e.is_deleted IS NOT NULL')->limit($options['limit']);
foreach ($q->execute() as $entity) {
//primary_extension automatically set upon save
$entity->save();
}
$db->commit();
} catch (Exception $e) {
$db->rollback();
throw $e;
}
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$db = Doctrine_Manager::connection();
$q = LsDoctrineQuery::create()->from('Entity e')->leftJoin('e.Alias a')->where('NOT EXISTS( SELECT id FROM alias a WHERE a.entity_id = e.id AND a.context IS NULL AND a.name = e.name)')->limit($options['limit']);
foreach ($q->execute() as $entity) {
try {
$db->beginTransaction();
$primary = $entity->primary_ext;
if ($primary == 'Person') {
$name = $entity->getExtensionObject('Person')->getFullName(false, $filterSuffix = true);
//change entity name
$entity->setEntityField('name', $name);
$entity->save();
} else {
$name = $entity->rawGet('name');
}
//create primary Alias
$a = new Alias();
$a->Entity = $entity;
$a->name = $name;
$a->is_primary = true;
$a->save();
//create another Alias if there's a nickname
if ($primary == 'Person' && $entity->name_nick) {
$a = new Alias();
$a->Entity = $entity;
$a->name = $primary == 'Person' ? $entity->name_nick . ' ' . $entity->name_last : $entity->name_nick;
$a->is_primary = false;
$a->save();
}
$db->commit();
} catch (Exception $e) {
$db->rollback();
throw $e;
}
}
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$time = microtime(true);
$this->init($arguments, $options);
$count = 0;
$this->printDebug("Matching OpenSecrets entities with OpenSecrets industry codes...");
if (in_array('Person', $this->types)) {
$persons = $this->getUnmatchedPersons();
foreach ($persons as $person) {
if ($this->isMatched($person['id'])) {
continue;
}
$this->printDebug("\nProcessing " . $person['name'] . " [" . $person['id'] . "]...");
$newCategories = OsPerson::updateCategories($person['id']);
foreach ($newCategories as $categoryId) {
$this->printDebug("+ added industry category: " . $this->getCategoryName($categoryId) . " [" . $categoryId . "]");
}
$count++;
}
}
if (in_array('Org', $this->types)) {
$orgs = $this->getUnmatchedOrgs();
foreach ($orgs as $org) {
if ($this->isMatched($org['id'])) {
continue;
}
if ($this->isSkippableEntity($org['id'])) {
continue;
}
$this->printDebug("\nProcessing " . $org['name'] . " [" . $org['id'] . "]...");
$newCategories = OsOrg::updateCategories($org['id'], $org['name'], $this->exactNameOverride);
foreach ($newCategories as $categoryId) {
$this->printDebug("+ added industry category: " . $this->getCategoryName($categoryId) . " [" . $categoryId . "]");
}
$count++;
}
}
print "\nMatched " . $count . " entities with OpenSecrets industry codes in " . (microtime(true) - $time) . " s\n";
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$db = Doctrine_Manager::connection();
//CREATE NEW PRIMARY ALIASES
$q = Doctrine_Query::create()->from('Entity e')->where('NOT EXISTS (SELECT id FROM Alias a WHERE a.entity_id = e.id AND a.is_primary = ?)', true)->limit($options['limit']);
foreach ($q->fetchArray() as $entity) {
$a = new Alias();
$a->name = $entity['name'];
$a->is_primary = true;
$a->entity_id = $entity['id'];
$a->save(false);
}
//REMOVE DUPLIATE PRIMARY ALIASES
$q = Doctrine_Query::create()->from('Alias a1')->where('a1.is_primary = ?', true)->andWhere('EXISTS (SELECT a2.id FROM Alias a2 WHERE a1.entity_id = a2.entity_id AND a2.is_primary = ? AND a1.id > a2.id)', true);
foreach ($q->execute() as $alias) {
$alias->delete();
}
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$this->s3 = new S3(sfConfig::get('app_amazon_access_key'), sfConfig::get('app_amazon_secret_key'));
$types = explode(",", $options['types']);
// hide strict errors
error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE);
foreach (array('js', 'css') as $type) {
if (in_array($type, $types)) {
$dir = sfConfig::get('sf_' . $type . '_dir');
if ($handle = opendir($dir)) {
/* This is the correct way to loop over the directory. */
while (false !== ($file = readdir($handle))) {
if (strpos($file, ".") == 0 || is_dir($dir . DIRECTORY_SEPARATOR . $file)) {
continue;
}
$this->uploadFile($type, $file, $options['check_first'], $options['debug_mode']);
}
closedir($handle);
}
}
}
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$this->init($arguments, $options);
$count = 0;
if ($options['debug_mode']) {
print "Importing OpenSecrets industry categories...\n";
}
//open file
$file = fopen($options['source_file'], 'r');
//process rows
while ($data = fgetcsv($file, 1000, "\t")) {
//skip header line
if ($data[0] == 'Catcode') {
continue;
}
if (count($data) < 5) {
if ($options['debug_mode']) {
print "Skipping line " . ($count + 1) . ": only " . count($data) . " fields...\n";
}
$count++;
continue;
}
$data = array_slice($data, 0, 6);
array_splice($data, 4, 1);
$data = array_map('trim', $data);
$this->insertData($data);
$count++;
if ($options['debug_mode']) {
print "Processed OpenSecrets industry category " . $data[0] . "\n";
flush();
}
}
print "Imported " . $count . " OpenSecrets industry categories\n";
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$db = Doctrine_Manager::connection();
//get scheduled emails
$q = LsDoctrineQuery::create()->from('ScheduledEmail se')->where('se.is_sent = ?', false)->limit($options['limit'])->orderBy('se.id');
foreach ($q->execute() as $email) {
$mailer = new Swift(new Swift_Connection_NativeMail());
if ($email->body_html) {
$message = new Swift_Message($email->subject);
$message->attach(new Swift_Message_Part($email->body_text));
$message->attach(new Swift_Message_Part($email->body_html, 'text/html'));
} else {
$message = new Swift_Message($email->subject, $email->body_text, 'text/plain');
}
$from_address = new Swift_Address($email->from_email, $email->from_name);
$to_address = new Swift_Address($email->to_email, $email->to_name);
if ($sent = $mailer->send($message, $to_address, $from_address)) {
$email->is_sent = true;
$email->save();
echo "Successfully sent scheduled email #" . $email->id . "\n";
echo " From: " . $email->from_email . "\n";
echo " To: " . $email->to_email . "\n";
echo " Subject: " . $email->subject . "\n";
echo "\n";
} else {
echo "Error sending scheduled email #" . $email->id . "\n";
echo "\n";
}
$mailer->disconnect();
}
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$db = Doctrine_Manager::connection();
//get entities with parent_ids
$sql = 'SELECT e.id, e.parent_id FROM entity e ' . 'LEFT JOIN entity p ON (p.id = e.parent_id) LEFT JOIN relationship r on (r.entity1_id = e.id AND r.entity2_id = e.parent_id AND r.category_id = ?)' . 'WHERE e.parent_id IS NOT NULL AND e.is_deleted <> 1 AND p.is_deleted <> 1 AND r.id IS NULL';
$stmt = $db->execute($sql, array(RelationshipTable::HIERARCHY_CATEGORY));
$entities = $stmt->fetchAll(PDO::FETCH_ASSOC);
//create hierarchy relationship for each entity
foreach ($entities as $entity) {
print "creating hierarchy relationship between entity " . $entity['id'] . " and entity " . $entity['parent_id'] . "\n";
//create relationship
$sql = 'INSERT into relationship (entity1_id,entity2_id,category_id,is_current,last_user_id,created_at,updated_at) VALUES (?,?,11,1,1,NOW(),NOW())';
$stmt = $db->execute($sql, array($entity['id'], $entity['parent_id']));
$lastId = $db->lastInsertId();
//create hierarchy
$sql = 'INSERT INTO hierarchy (relationship_id) VALUES (?)';
$stmt = $db->execute($sql, array($lastId));
}
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
//set start time
$time = microtime(true);
//connect to raw database
$this->init($arguments, $options);
//open file
$file = fopen(sfConfig::get('sf_root_dir') . '/data/opensecrets/indivs' . substr($options['cycle'], -2) . '.txt', 'r');
//set counters
$line = 0;
$this->added = 0;
$offset = $options['offset'];
$limit = $offset + $options['limit'];
/*
//determine how many rows already processed
$sql = 'SELECT COUNT(*) FROM os_donation WHERE cycle = ?';
$stmt = $this->db->execute($sql, array($options['cycle']));
$count = $stmt->fetch(PDO::FETCH_COLUMN);
*/
//skip to offset line
while ($line < $offset) {
$data = fgetcsv($file, 1000, ",", "|");
$line++;
}
//process remaining rows
while ($line < $limit && ($data = fgetcsv($file, 1000, ",", "|"))) {
$line++;
if ($this->checkDups) {
if ($this->isDonationDuplicate($data[0], $data[1])) {
if ($options['debug_mode']) {
print "[line " . $line . "] " . $data[0] . " donation record " . $data[1] . " already exists! Skipping...\n";
}
continue;
} else {
if ($options['debug_mode']) {
print "[line " . $line . "] " . $data[0] . " donation record " . $data[1] . " is NEW! Adding...\n";
}
$this->added++;
}
}
if (count($data) == 23) {
$old = $data;
array_splice($data, 19, 0, "");
}
if (count($data) != 24) {
$data = array_slice($data, 0, 2);
for ($n = 0; $n < 22; $n++) {
$data[] = null;
}
}
//convert date format
if ($data[8]) {
$data[8] = date('Y-m-d', strtotime($data[8]));
}
$data = array_merge($data, self::parseDonorName($data));
//trim strings and replace blank strings with NULL
$func = function ($value) {
$value = trim($value);
return $value ? $value : NULL;
};
$data = array_map($func, $data);
$this->insertData($data);
if ($options['debug_mode']) {
print "Processed OpenSecrets " . $options['cycle'] . " donation record " . $data[1] . "\n";
flush();
}
}
$duration = microtime(true) - $time;
print $this->added . " total OpenSecrets " . $options['cycle'] . " donation records added in " . $duration . " s\n";
//DONE
LsCli::beep();
}
protected function execute($arguments = array(), $options = array())
{
$configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
$databaseManager = new sfDatabaseManager($configuration);
$databaseManager->initialize($configuration);
$startTime = microtime(true);
$db = Doctrine_Manager::connection();
$urls = array();
switch ($options['type']) {
case 'all':
$types = array('static', 'entity', 'relationship', 'list', 'user', 'group');
$tmpArguments = $arguments;
$tmpOptions = $options;
foreach ($types as $type) {
$this->run(array($tmpArguments['application']), array('env=' . $tmpOptions['env'], 'limit=' . $tmpOptions['limit'], 'type=' . $type));
}
if (count($this->fileNames)) {
$this->writeSitemapIndex();
}
return;
case 'static':
//static urls
$urls = array('/', '/lists', '/modification/latest', '/notes', '/groups', '/guide', '/faq', '/howto', '/videos', '/contact', '/join', '/about', '/features', '/team', '/press', '/disclaimer');
break;
case 'entity':
//get entity urls
$q = LsDoctrineQuery::create()->select("CONCAT('/', LOWER(e.primary_ext), '/', e.id, '/', REPLACE(REPLACE(e.name, ' ', '_'), '/', '~'))")->from('Entity e')->where('e.is_deleted = 0');
$stmt = $db->execute($q->getSqlQuery());
$entityUrls = $stmt->fetchAll(PDO::FETCH_COLUMN);
$patterns = array('', '/references', '/notes', '/images', '/interlocks', '/giving');
foreach ($entityUrls as $u) {
foreach ($patterns as $p) {
$urls[] = $u . $p;
}
}
break;
case 'relationship':
//get relationship urls
$q = LsDoctrineQuery::create()->select('r.id')->from('Relationship r')->where('r.is_deleted = 0');
$stmt = $db->execute($q->getSqlQuery());
$relIds = $stmt->fetchAll(PDO::FETCH_COLUMN);
$patterns = array('view', 'references', 'notes');
foreach ($relIds as $id) {
foreach ($patterns as $p) {
$urls[] = '/relationship/' . $p . '/id/' . $id;
}
}
break;
case 'list':
//get list urls
$q = LsDoctrineQuery::create()->select("CONCAT('/', l.id, '/', REPLACE(REPLACE(l.name, ' ', '_'), '/', '~'))")->from('LsList l')->where('l.is_deleted = 0');
$stmt = $db->execute($q->getSqlQuery());
$listUrls = $stmt->fetchAll(PDO::FETCH_COLUMN);
$patterns = array('', '/references', '/notes', '/members', '/interlocks', '/giving', '/funding');
foreach ($listUrls as $u) {
foreach ($patterns as $p) {
$urls[] = '/list' . $u . $p;
}
}
break;
case 'user':
//get user urls
$q = LsDoctrineQuery::create()->select('p.public_name')->from('sfGuardUserProfile p')->where('p.is_visible = 1 AND p.is_confirmed = 1')->setHydrationMode(Doctrine::HYDRATE_NONE);
$stmt = $db->execute($q->getSqlQuery());
$names = $stmt->fetchAll(PDO::FETCH_COLUMN);
$patterns = array('', '/groups');
foreach ($names as $name) {
foreach ($patterns as $p) {
$urls[] = '/user/' . $name . $p;
}
}
break;
case 'group':
//get group urls
$q = LsDoctrineQuery::create()->select('g.name')->from('sfGuardGroup g')->where('g.is_working = 1 AND g.is_private = 0')->setHydrationMode(Doctrine::HYDRATE_NONE);
$stmt = $db->execute($q->getSqlQuery());
$names = $stmt->fetchAll(PDO::FETCH_COLUMN);
foreach ($names as $name) {
$urls[] = '/group/' . $name;
}
break;
}
//break urls into sets of 50,000
$num = ceil(count($urls) / 50000);
$urlSets = array();
for ($n = 0; $n < $num; $n++) {
$urlSets[] = array_slice($urls, $n * 50000, 50000);
}
print "Found " . count($urls) . " " . $options['type'] . " URLs; writing to " . count($urlSets) . " Sitemap files...\n";
//generate sitemap XML file for each set
foreach ($urlSets as $key => $set) {
$xml = '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
$xml .= '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">' . "\n";
foreach ($set as $url) {
$xml .= " <url>\n";
$xml .= " <loc>" . htmlspecialchars($this->urlBase . $url) . "</loc>\n";
$xml .= " </url>\n";
}
$xml .= "</urlset>";
//write to file in /web
$fileName = $options['type'] . '-sitemap.' . ($key + 1) . '.xml';
$f = fopen(sfConfig::get('sf_web_dir') . '/' . $fileName, 'w');
fwrite($f, $xml);
fclose($f);
$this->fileNames[] = $fileName;
print "+ Wrote " . count($set) . " URLs to sitemap file: " . $fileName . "\n";
}
//DONE
LsCli::beep();
}