protected function execute($arguments = array(), $options = array())
 {
     $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
     $databaseManager = new sfDatabaseManager($configuration);
     $databaseManager->initialize($configuration);
     $this->db = Doctrine_Manager::connection();
     $this->s3 = new S3(sfConfig::get('app_amazon_access_key'), sfConfig::get('app_amazon_secret_key'));
     // hide strict errors
     error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE);
     $sql = "SELECT DISTINCT(r.source) FROM reference r ORDER BY id ASC LIMIT " . $options["limit"] . " OFFSET " . $options["offset"];
     $stmt = $this->db->execute($sql);
     $sources = $stmt->fetchAll(PDO::FETCH_COLUMN);
     foreach ($sources as $source) {
         if (!$options["localdir"]) {
             $s3path = ReferenceTable::generateS3path($source);
             if (!$options["overwrite"] && $this->s3->getObjectInfo(sfConfig::get('app_amazon_s3_bucket'), $s3path) !== false) {
                 print "ALREADY UPLOADED: " . $s3path . "\n";
                 continue;
             }
         }
         $this->writeTmpFile($source, $options["debug_mode"], $options["localdir"]);
         if (!$options["localdir"]) {
             $this->uploadTmpFile($source, $options["overwrite"], $options["debug_mode"]);
             if (unlink($this->getLocalPath($source, $options["local_dir"])) && $options["debug_mode"]) {
                 print "REMOVED LOCAL: " . $source . " [" . sha1($source) . "]\n";
             }
         }
         print "----------------------------------------------------------------\n";
     }
     //DONE
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     //set start time
     $time = microtime(true);
     //connect to raw database
     $this->init($arguments, $options);
     $count = 0;
     if ($options['debug_mode']) {
         print "Processing OpenSecrets lobbying agency records...\n";
     }
     //open file
     $file = fopen(sfConfig::get('sf_root_dir') . '/data/opensecrets/lob_agency.txt', 'r');
     //process rows
     while ($data = fgetcsv($file, 1000, ',', '|')) {
         $this->insertData($data);
         $count++;
         if ($options['debug_mode']) {
             print "Processed OpenSecrets lobbying agency for lobbying record " . $data[0] . ": " . $data[2] . "\n";
             flush();
         }
     }
     $duration = microtime(true) - $time;
     print $count . " OpenSecrets lobbying agency records processed in " . $duration . " s\n";
     //DONE
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
     $databaseManager = new sfDatabaseManager($configuration);
     $databaseManager->initialize($configuration);
     $db = Doctrine_Manager::connection();
     $models = array('Entity', 'Relationship', 'LsList');
     foreach ($models as $model) {
         $modelAlias = strtolower($model);
         $updateAlias = $model == 'LsList' ? 'ls_list' : $modelAlias;
         //get records to update
         $q = LsDoctrineQuery::create()->select('id')->from($model . ' ' . $modelAlias)->where($modelAlias . '.last_user_id IS NULL')->limit($options['limit'])->setHydrationMode(Doctrine::HYDRATE_NONE);
         if (!count($rows = $q->execute())) {
             //nothing to update, go to next model
             continue;
         }
         foreach ($rows as $row) {
             $id = $row[0];
             //get last_user_id
             $result = LsDoctrineQuery::create()->select('m.user_id')->from('Modification m')->where('m.object_model = ? AND m.object_id = ?', array($model, $id))->orderBy('m.id DESC')->setHydrationMode(Doctrine::HYDRATE_NONE)->fetchOne();
             if ($lastUserId = $result[0]) {
                 $query = 'UPDATE ' . $updateAlias . ' SET last_user_id=? WHERE id=?';
                 //use PDO for speed
                 $db->execute($query, array($lastUserId, $id));
             } else {
                 throw new Exception("Couldn't find last_user_id for " . $model . ' #' . $id);
             }
         }
         //only update records of one model at a time
         break;
     }
     //DONE
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
     $databaseManager = new sfDatabaseManager($configuration);
     $databaseManager->initialize($configuration);
     $db = Doctrine_Manager::connection();
     //get person entities with all-caps names
     $sql = 'SELECT e.id, e.name FROM entity e ' . 'WHERE e.name <> \'\' AND e.primary_ext = ? AND CAST(UPPER(e.name) AS BINARY) = CAST(e.name AS BINARY)';
     $stmt = $db->execute($sql, array('Person'));
     $names = $stmt->fetchAll(PDO::FETCH_ASSOC);
     foreach ($names as $ary) {
         $new = PersonTable::nameizePersonName($ary['name']);
         if ($new != $ary['name']) {
             $sql = 'UPDATE entity SET name = ? WHERE id = ?';
             $stmt = $db->execute($sql, array($new, $ary['id']));
             print "Changed Entity name " . $ary['name'] . " to " . $new . "\n";
         }
     }
     //get aliases with all-caps names
     $sql = 'SELECT a.id, a.name FROM alias a LEFT JOIN entity e ON (e.id = a.entity_id) ' . 'WHERE a.name <> \'\' AND a.is_primary = 1 AND e.primary_ext = ? AND ' . 'CAST(UPPER(a.name) AS BINARY) = CAST(a.name AS BINARY)';
     $stmt = $db->execute($sql, array('Person'));
     $names = $stmt->fetchAll(PDO::FETCH_ASSOC);
     foreach ($names as $ary) {
         $new = PersonTable::nameizePersonName($ary['name']);
         if ($new != $ary['name']) {
             $sql = 'UPDATE alias SET name = ? WHERE id = ?';
             $stmt = $db->execute($sql, array($new, $ary['id']));
             print "Changed Alias " . $ary['name'] . " to " . $new . "\n";
         }
     }
     //DONE
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
     $databaseManager = new sfDatabaseManager($configuration);
     $databaseManager->initialize($configuration);
     $this->debugMode = $options['debug_mode'];
     $this->db = Doctrine_Manager::connection();
     $this->s3 = new S3(sfConfig::get('app_amazon_access_key'), sfConfig::get('app_amazon_secret_key'));
     // hide strict errors
     error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE);
     //get array of active entity image filenames
     if ($options['list_id']) {
         $sql = "SELECT i.id, i.filename, i.url, i.entity_id, e.name, e.primary_ext " . "FROM ls_list_entity le " . "LEFT JOIN image i ON (i.entity_id = le.entity_id) " . "LEFT JOIN entity e ON (e.id = le.entity_id) " . "WHERE le.list_id = ? AND le.is_deleted = 0 " . "AND i.is_deleted = 0 AND i.has_square = 0 " . "ORDER BY id DESC LIMIT " . $options['limit'];
         $params = array($options['list_id']);
     } else {
         $sql = "SELECT i.id, i.filename FROM image i " . "WHERE is_deleted = 0 AND has_square = 0 " . "ORDER BY id DESC LIMIT " . $options['limit'];
         $params = array();
     }
     $stmt = $this->db->execute($sql, $params);
     $images = $stmt->fetchAll(PDO::FETCH_ASSOC);
     $count = count($images);
     foreach ($images as $image) {
         $this->printDebug("Processing image {$image['id']} belonging to entity {$image['entity_id']}...");
         if ($this->downloadLarge($image['filename'])) {
             $this->printDebug("Downloaded large image from S3: " . $s3Path);
         } else {
             $s3Path = ImageTable::generateS3Url(ImageTable::getPath($image['filename'], 'large'));
             $this->printDebug("Couldn't download large image from S3: " . $s3Path);
             if ($image['url']) {
                 if ($this->downloadToTmp($image['url'], $image['filename'])) {
                     $this->printDebug("Downloaded original image: " . $image['url']);
                 } else {
                     $this->printDebug("Couldn't download original image: " . $image['url']);
                     if ($this->downloadFromGoogle($image['name'])) {
                         $this->printDebug("Downloaded new image of {$image['name']} from google");
                     } else {
                         $count--;
                         continue;
                     }
                 }
             } else {
                 $count--;
                 continue;
             }
         }
         if (!$this->createSquare($image['filename'], $options['size'])) {
             $this->printDebug("Coudln't create square image: {$image['filename']}");
             $count--;
             continue;
         }
         if ($this->uploadFile($image['filename'], $options['check_first'], $options['debug_mode'])) {
             $this->recordSquare($image['id']);
         }
         $count--;
         print $count . " images remaining...\n";
     }
     //DONE
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
     $databaseManager = new sfDatabaseManager($configuration);
     $databaseManager->initialize($configuration);
     $this->db = Doctrine_Manager::connection();
     $this->s3 = new S3(sfConfig::get('app_amazon_access_key'), sfConfig::get('app_amazon_secret_key'));
     $types = explode(",", $options['types']);
     // hide strict errors
     error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE);
     if (count(array_intersect($types, array("profile", "small", "large")))) {
         //get array of active entity image filenames
         $sql = "SELECT i.filename FROM image i WHERE is_deleted = 0 ORDER BY id DESC LIMIT " . $options['limit'];
         $stmt = $this->db->execute($sql);
         $images = $stmt->fetchAll(PDO::FETCH_ASSOC);
         $count = count($images);
         foreach ($images as $image) {
             foreach ($types as $type) {
                 if (in_array($type, array("system", "user"))) {
                     continue;
                 }
                 $this->uploadFile($type, $image['filename'], $options['check_first'], $options['debug_mode']);
             }
             $count--;
             print $count . " entities remaining...\n";
         }
     }
     if (in_array("system", $types)) {
         //do system images
         if ($handle = opendir(sfConfig::get('sf_image_dir') . "/system")) {
             /* This is the correct way to loop over the directory. */
             while (false !== ($file = readdir($handle))) {
                 if (strpos($file, ".") == 0) {
                     continue;
                 }
                 $this->uploadFile("system", $file, $options['check_first'], $options['debug_mode']);
             }
             closedir($handle);
         }
     }
     if (in_array("user", $types)) {
         //get array of active entity image filenames
         $sql = "SELECT p.filename FROM sf_guard_user_profile p WHERE p.filename IS NOT NULL";
         $stmt = $this->db->execute($sql);
         $files = $stmt->fetchAll(PDO::FETCH_COLUMN);
         $count = count($files);
         foreach ($files as $file) {
             foreach (array("profile", "small") as $type) {
                 $this->uploadFile($type, $file, $options['check_first'], $options['debug_mode']);
             }
             $count--;
             print $count . " users remaining...\n";
         }
     }
     //DONE
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
     $databaseManager = new sfDatabaseManager($configuration);
     $databaseManager->initialize($configuration);
     $this->db = Doctrine_Manager::connection();
     //get last processed merge modification
     if (!$this->hasMeta('log', 'last_modification_id')) {
         $lastModId = 0;
     } else {
         $lastModId = $this->getMeta('log', 'last_modification_id');
     }
     //get merges since last processed merge modification
     $sql = 'SELECT m.* FROM modification m WHERE m.id > ? AND m.object_model = ? AND m.is_merge = 1 AND m.merge_object_id IS NOT NULL ORDER BY m.id DESC';
     $stmt = $this->db->execute($sql, array($lastModId, 'Entity'));
     $mods = $stmt->fetchAll(PDO::FETCH_ASSOC);
     //for each merge, update any links involving the merged entity
     foreach ($mods as $mod) {
         if ($options['debug_mode']) {
             print "Updating links for merged entity " . $mod['object_id'] . " (to " . $mod['merge_object_id'] . ")\n";
         }
         $sql = 'UPDATE link l SET entity1_id = ? WHERE entity1_id = ?';
         $stmt = $this->db->execute($sql, array($mod['merge_object_id'], $mod['object_id']));
         $sql = 'UPDATE link l SET entity2_id = ? WHERE entity2_id = ?';
         $stmt = $this->db->execute($sql, array($mod['merge_object_id'], $mod['object_id']));
     }
     //save highest merge modification for next time
     if (count($mods)) {
         $this->saveMeta('log', 'last_modification_id', $mods[0]['id']);
     }
     //get relationships with is_deleted = 0 and without links
     $sql = 'SELECT r.* FROM relationship r LEFT JOIN link l ON (r.id = l.relationship_id) WHERE r.is_deleted = 0 AND l.id IS NULL ORDER BY r.id LIMIT ' . $options['limit'];
     $stmt = $this->db->execute($sql);
     $rels = $stmt->fetchAll(PDO::FETCH_ASSOC);
     foreach ($rels as $rel) {
         if ($options['debug_mode']) {
             print "Creating Links for Relationship " . $rel['id'] . "\n";
         }
         $this->createLinks($rel);
     }
     //get relationships with is_deleted = 1 and with links
     $sql = 'SELECT r.* FROM relationship r LEFT JOIN link l ON (r.id = l.relationship_id) WHERE r.is_deleted = 1 AND l.id IS NOT NULL GROUP BY r.id LIMIT ' . $options['limit'];
     $stmt = $this->db->execute($sql);
     $rels = $stmt->fetchAll(PDO::FETCH_ASSOC);
     foreach ($rels as $rel) {
         if ($options['debug_mode']) {
             print "Deleting Links for Relationship " . $rel['id'] . "\n";
         }
         $this->deleteLinks($rel);
     }
     //DONE
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     //set start time
     $time = microtime(true);
     //connect to raw database
     $this->init($arguments, $options);
     $count = 0;
     foreach ($this->cycles as $cycle) {
         if ($options['debug_mode']) {
             print "Processing OpenSecrets candidates from " . $cycle . " cycle...\n";
         }
         //open file
         $file = fopen(sfConfig::get('sf_root_dir') . '/data/opensecrets/cands' . substr($cycle, -2) . '.txt', 'r');
         //process rows
         while ($data = fgetcsv($file, 1000, ",", "|")) {
             if ($count >= $this->limit) {
                 if ($options['debug_mode']) {
                     print "*** Import limit reached; exiting...\n";
                 }
                 break;
             }
             if ($this->isCandidateDuplicate($data[1], $data[2], $data[0])) {
                 if ($options['debug_mode']) {
                     print "- Candidate " . $data[2] . " already exists; skipping...\n";
                 }
                 continue;
             }
             if (count($data) != 12) {
                 if ($options['debug_mode']) {
                     print "* Candidate " . $data[2] . " has " . count($data) . " fields; adjusting...\n";
                 }
                 $data = array_slice($data, 0, 2);
                 for ($n = 0; $n < 10; $n++) {
                     $data[] = null;
                 }
             }
             $data = $this->parseCandidateName($data);
             $this->insertData($data);
             $count++;
             if ($options['debug_mode']) {
                 print "Processed OpenSecrets " . $cycle . " candidate record " . $data[2] . "\n";
                 flush();
             }
         }
     }
     $duration = microtime(true) - $time;
     print $count . " OpenSecrets candidate records processed in " . $duration . " s\n";
     //DONE
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
     $databaseManager = new sfDatabaseManager($configuration);
     $databaseManager->initialize($configuration);
     //set start time
     $time = microtime(true);
     //connect to raw database
     $this->init($arguments, $options);
     $count = 0;
     foreach ($this->cycles as $cycle) {
         if ($options['debug_mode']) {
             print "Processing OpenSecrets committees from " . $cycle . " cycle...\n";
         }
         //open file
         $file = fopen(sfConfig::get('sf_root_dir') . '/data/opensecrets/cmtes' . substr($cycle, -2) . '.txt', 'r');
         //process rows
         while ($data = fgetcsv($file, 1000, ",", "|")) {
             if ($count >= $this->limit) {
                 if ($options['debug_mode']) {
                     print "*** Import limit reached; exiting...\n";
                 }
                 break;
             }
             if ($this->isCommitteeDuplicate($data[0], $data[1])) {
                 if ($options['debug_mode']) {
                     print "- Committee " . $data[1] . " already exists for " . $data[0] . " cycle; skipping...\n";
                 }
                 continue;
             }
             if (count($data) != 14) {
                 $data = array_slice($data, 0, 3);
                 for ($n = 0; $n < 11; $n++) {
                     $data[] = null;
                 }
             }
             $this->insertData($data);
             $count++;
             if ($options['debug_mode']) {
                 print "Processed OpenSecrets " . $cycle . " committee record " . $data[1] . "\n";
                 flush();
             }
         }
     }
     $duration = microtime(true) - $time;
     print $count . " OpenSecrets committee records processed in " . $duration . " s\n";
     //DONE
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
     $databaseManager = new sfDatabaseManager($configuration);
     $databaseManager->initialize($configuration);
     //set start time
     $time = microtime(true);
     //connect to raw database
     $this->init($arguments, $options);
     //determine how many rows alrady processed
     $sql = 'SELECT COUNT(*) FROM os_pac_donation WHERE cycle = ?';
     $stmt = $this->db->execute($sql, array($options['cycle']));
     $count = $stmt->fetch(PDO::FETCH_COLUMN);
     //open file
     $file = fopen(sfConfig::get('sf_root_dir') . '/data/opensecrets/pacs' . substr($options['cycle'], -2) . '.csv', 'r');
     //skip to last line in file
     $line = 0;
     while ($line < $count) {
         fgetcsv($file, 1000);
         $line++;
     }
     //process remaining rows
     $limit = $line + $options['limit'];
     while ($line < $limit && ($data = fgetcsv($file, 1000))) {
         if (count($data) != 10) {
             die('Unepexted number of columns on line ' . $line . ': ' . count($data));
         }
         //convert date format
         if ($data[5]) {
             $data[5] = date('Y-m-d', strtotime($data[8]));
         }
         $this->insertData($data);
         $line++;
         if ($options['debug_mode']) {
             print "Processed OpenSecrets " . $options['cycle'] . " PAC donation record " . $data[1] . "\n";
             flush();
         }
     }
     $duration = microtime(true) - $time;
     print $limit . " total OpenSecrets " . $options['cycle'] . " PAC donation records processed in " . $duration . " s\n";
     //DONE
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
     $databaseManager = new sfDatabaseManager($configuration);
     $databaseManager->initialize($configuration);
     $db = Doctrine_Manager::connection();
     try {
         $db->beginTransaction();
         $q = LsDoctrineQuery::create()->from('Entity e')->where('e.primary_ext IS NULL')->andWhere('e.is_deleted IS NOT NULL')->limit($options['limit']);
         foreach ($q->execute() as $entity) {
             //primary_extension automatically set upon save
             $entity->save();
         }
         $db->commit();
     } catch (Exception $e) {
         $db->rollback();
         throw $e;
     }
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
     $databaseManager = new sfDatabaseManager($configuration);
     $databaseManager->initialize($configuration);
     $db = Doctrine_Manager::connection();
     $q = LsDoctrineQuery::create()->from('Entity e')->leftJoin('e.Alias a')->where('NOT EXISTS( SELECT id FROM alias a WHERE a.entity_id = e.id AND a.context IS NULL AND a.name = e.name)')->limit($options['limit']);
     foreach ($q->execute() as $entity) {
         try {
             $db->beginTransaction();
             $primary = $entity->primary_ext;
             if ($primary == 'Person') {
                 $name = $entity->getExtensionObject('Person')->getFullName(false, $filterSuffix = true);
                 //change entity name
                 $entity->setEntityField('name', $name);
                 $entity->save();
             } else {
                 $name = $entity->rawGet('name');
             }
             //create primary Alias
             $a = new Alias();
             $a->Entity = $entity;
             $a->name = $name;
             $a->is_primary = true;
             $a->save();
             //create another Alias if there's a nickname
             if ($primary == 'Person' && $entity->name_nick) {
                 $a = new Alias();
                 $a->Entity = $entity;
                 $a->name = $primary == 'Person' ? $entity->name_nick . ' ' . $entity->name_last : $entity->name_nick;
                 $a->is_primary = false;
                 $a->save();
             }
             $db->commit();
         } catch (Exception $e) {
             $db->rollback();
             throw $e;
         }
     }
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     $time = microtime(true);
     $this->init($arguments, $options);
     $count = 0;
     $this->printDebug("Matching OpenSecrets entities with OpenSecrets industry codes...");
     if (in_array('Person', $this->types)) {
         $persons = $this->getUnmatchedPersons();
         foreach ($persons as $person) {
             if ($this->isMatched($person['id'])) {
                 continue;
             }
             $this->printDebug("\nProcessing " . $person['name'] . " [" . $person['id'] . "]...");
             $newCategories = OsPerson::updateCategories($person['id']);
             foreach ($newCategories as $categoryId) {
                 $this->printDebug("+ added industry category: " . $this->getCategoryName($categoryId) . " [" . $categoryId . "]");
             }
             $count++;
         }
     }
     if (in_array('Org', $this->types)) {
         $orgs = $this->getUnmatchedOrgs();
         foreach ($orgs as $org) {
             if ($this->isMatched($org['id'])) {
                 continue;
             }
             if ($this->isSkippableEntity($org['id'])) {
                 continue;
             }
             $this->printDebug("\nProcessing " . $org['name'] . " [" . $org['id'] . "]...");
             $newCategories = OsOrg::updateCategories($org['id'], $org['name'], $this->exactNameOverride);
             foreach ($newCategories as $categoryId) {
                 $this->printDebug("+ added industry category: " . $this->getCategoryName($categoryId) . " [" . $categoryId . "]");
             }
             $count++;
         }
     }
     print "\nMatched " . $count . " entities with OpenSecrets industry codes in " . (microtime(true) - $time) . " s\n";
     //DONE
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
     $databaseManager = new sfDatabaseManager($configuration);
     $databaseManager->initialize($configuration);
     $db = Doctrine_Manager::connection();
     //CREATE NEW PRIMARY ALIASES
     $q = Doctrine_Query::create()->from('Entity e')->where('NOT EXISTS (SELECT id FROM Alias a WHERE a.entity_id = e.id AND a.is_primary = ?)', true)->limit($options['limit']);
     foreach ($q->fetchArray() as $entity) {
         $a = new Alias();
         $a->name = $entity['name'];
         $a->is_primary = true;
         $a->entity_id = $entity['id'];
         $a->save(false);
     }
     //REMOVE DUPLIATE PRIMARY ALIASES
     $q = Doctrine_Query::create()->from('Alias a1')->where('a1.is_primary = ?', true)->andWhere('EXISTS (SELECT a2.id FROM Alias a2 WHERE a1.entity_id = a2.entity_id AND a2.is_primary = ? AND a1.id > a2.id)', true);
     foreach ($q->execute() as $alias) {
         $alias->delete();
     }
     //DONE
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     $this->s3 = new S3(sfConfig::get('app_amazon_access_key'), sfConfig::get('app_amazon_secret_key'));
     $types = explode(",", $options['types']);
     // hide strict errors
     error_reporting(E_ERROR | E_WARNING | E_PARSE | E_NOTICE);
     foreach (array('js', 'css') as $type) {
         if (in_array($type, $types)) {
             $dir = sfConfig::get('sf_' . $type . '_dir');
             if ($handle = opendir($dir)) {
                 /* This is the correct way to loop over the directory. */
                 while (false !== ($file = readdir($handle))) {
                     if (strpos($file, ".") == 0 || is_dir($dir . DIRECTORY_SEPARATOR . $file)) {
                         continue;
                     }
                     $this->uploadFile($type, $file, $options['check_first'], $options['debug_mode']);
                 }
                 closedir($handle);
             }
         }
     }
     //DONE
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     $this->init($arguments, $options);
     $count = 0;
     if ($options['debug_mode']) {
         print "Importing OpenSecrets industry categories...\n";
     }
     //open file
     $file = fopen($options['source_file'], 'r');
     //process rows
     while ($data = fgetcsv($file, 1000, "\t")) {
         //skip header line
         if ($data[0] == 'Catcode') {
             continue;
         }
         if (count($data) < 5) {
             if ($options['debug_mode']) {
                 print "Skipping line " . ($count + 1) . ": only " . count($data) . " fields...\n";
             }
             $count++;
             continue;
         }
         $data = array_slice($data, 0, 6);
         array_splice($data, 4, 1);
         $data = array_map('trim', $data);
         $this->insertData($data);
         $count++;
         if ($options['debug_mode']) {
             print "Processed OpenSecrets industry category " . $data[0] . "\n";
             flush();
         }
     }
     print "Imported " . $count . " OpenSecrets industry categories\n";
     //DONE
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
     $databaseManager = new sfDatabaseManager($configuration);
     $databaseManager->initialize($configuration);
     $db = Doctrine_Manager::connection();
     //get scheduled emails
     $q = LsDoctrineQuery::create()->from('ScheduledEmail se')->where('se.is_sent = ?', false)->limit($options['limit'])->orderBy('se.id');
     foreach ($q->execute() as $email) {
         $mailer = new Swift(new Swift_Connection_NativeMail());
         if ($email->body_html) {
             $message = new Swift_Message($email->subject);
             $message->attach(new Swift_Message_Part($email->body_text));
             $message->attach(new Swift_Message_Part($email->body_html, 'text/html'));
         } else {
             $message = new Swift_Message($email->subject, $email->body_text, 'text/plain');
         }
         $from_address = new Swift_Address($email->from_email, $email->from_name);
         $to_address = new Swift_Address($email->to_email, $email->to_name);
         if ($sent = $mailer->send($message, $to_address, $from_address)) {
             $email->is_sent = true;
             $email->save();
             echo "Successfully sent scheduled email #" . $email->id . "\n";
             echo "  From: " . $email->from_email . "\n";
             echo "  To: " . $email->to_email . "\n";
             echo "  Subject: " . $email->subject . "\n";
             echo "\n";
         } else {
             echo "Error sending scheduled email #" . $email->id . "\n";
             echo "\n";
         }
         $mailer->disconnect();
     }
     //DONE
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
     $databaseManager = new sfDatabaseManager($configuration);
     $databaseManager->initialize($configuration);
     $db = Doctrine_Manager::connection();
     //get entities with parent_ids
     $sql = 'SELECT e.id, e.parent_id FROM entity e ' . 'LEFT JOIN entity p ON (p.id = e.parent_id) LEFT JOIN relationship r on (r.entity1_id = e.id AND r.entity2_id = e.parent_id AND r.category_id = ?)' . 'WHERE e.parent_id IS NOT NULL AND e.is_deleted <> 1 AND p.is_deleted <> 1 AND r.id IS NULL';
     $stmt = $db->execute($sql, array(RelationshipTable::HIERARCHY_CATEGORY));
     $entities = $stmt->fetchAll(PDO::FETCH_ASSOC);
     //create hierarchy relationship for each entity
     foreach ($entities as $entity) {
         print "creating hierarchy relationship between entity " . $entity['id'] . " and entity " . $entity['parent_id'] . "\n";
         //create relationship
         $sql = 'INSERT into relationship (entity1_id,entity2_id,category_id,is_current,last_user_id,created_at,updated_at) VALUES (?,?,11,1,1,NOW(),NOW())';
         $stmt = $db->execute($sql, array($entity['id'], $entity['parent_id']));
         $lastId = $db->lastInsertId();
         //create hierarchy
         $sql = 'INSERT INTO hierarchy (relationship_id) VALUES (?)';
         $stmt = $db->execute($sql, array($lastId));
     }
     //DONE
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
     $databaseManager = new sfDatabaseManager($configuration);
     $databaseManager->initialize($configuration);
     //set start time
     $time = microtime(true);
     //connect to raw database
     $this->init($arguments, $options);
     //open file
     $file = fopen(sfConfig::get('sf_root_dir') . '/data/opensecrets/indivs' . substr($options['cycle'], -2) . '.txt', 'r');
     //set counters
     $line = 0;
     $this->added = 0;
     $offset = $options['offset'];
     $limit = $offset + $options['limit'];
     /*    
         //determine how many rows already processed
         $sql = 'SELECT COUNT(*) FROM os_donation WHERE cycle = ?';
         $stmt = $this->db->execute($sql, array($options['cycle']));
         $count = $stmt->fetch(PDO::FETCH_COLUMN);
     */
     //skip to offset line
     while ($line < $offset) {
         $data = fgetcsv($file, 1000, ",", "|");
         $line++;
     }
     //process remaining rows
     while ($line < $limit && ($data = fgetcsv($file, 1000, ",", "|"))) {
         $line++;
         if ($this->checkDups) {
             if ($this->isDonationDuplicate($data[0], $data[1])) {
                 if ($options['debug_mode']) {
                     print "[line " . $line . "]  " . $data[0] . " donation record " . $data[1] . " already exists! Skipping...\n";
                 }
                 continue;
             } else {
                 if ($options['debug_mode']) {
                     print "[line " . $line . "]  " . $data[0] . " donation record " . $data[1] . " is NEW! Adding...\n";
                 }
                 $this->added++;
             }
         }
         if (count($data) == 23) {
             $old = $data;
             array_splice($data, 19, 0, "");
         }
         if (count($data) != 24) {
             $data = array_slice($data, 0, 2);
             for ($n = 0; $n < 22; $n++) {
                 $data[] = null;
             }
         }
         //convert date format
         if ($data[8]) {
             $data[8] = date('Y-m-d', strtotime($data[8]));
         }
         $data = array_merge($data, self::parseDonorName($data));
         //trim strings and replace blank strings with NULL
         $func = function ($value) {
             $value = trim($value);
             return $value ? $value : NULL;
         };
         $data = array_map($func, $data);
         $this->insertData($data);
         if ($options['debug_mode']) {
             print "Processed OpenSecrets " . $options['cycle'] . " donation record " . $data[1] . "\n";
             flush();
         }
     }
     $duration = microtime(true) - $time;
     print $this->added . " total OpenSecrets " . $options['cycle'] . " donation records added in " . $duration . " s\n";
     //DONE
     LsCli::beep();
 }
 protected function execute($arguments = array(), $options = array())
 {
     $configuration = ProjectConfiguration::getApplicationConfiguration($arguments['application'], $options['env'], true);
     $databaseManager = new sfDatabaseManager($configuration);
     $databaseManager->initialize($configuration);
     $startTime = microtime(true);
     $db = Doctrine_Manager::connection();
     $urls = array();
     switch ($options['type']) {
         case 'all':
             $types = array('static', 'entity', 'relationship', 'list', 'user', 'group');
             $tmpArguments = $arguments;
             $tmpOptions = $options;
             foreach ($types as $type) {
                 $this->run(array($tmpArguments['application']), array('env=' . $tmpOptions['env'], 'limit=' . $tmpOptions['limit'], 'type=' . $type));
             }
             if (count($this->fileNames)) {
                 $this->writeSitemapIndex();
             }
             return;
         case 'static':
             //static urls
             $urls = array('/', '/lists', '/modification/latest', '/notes', '/groups', '/guide', '/faq', '/howto', '/videos', '/contact', '/join', '/about', '/features', '/team', '/press', '/disclaimer');
             break;
         case 'entity':
             //get entity urls
             $q = LsDoctrineQuery::create()->select("CONCAT('/', LOWER(e.primary_ext), '/', e.id, '/', REPLACE(REPLACE(e.name, ' ', '_'), '/', '~'))")->from('Entity e')->where('e.is_deleted = 0');
             $stmt = $db->execute($q->getSqlQuery());
             $entityUrls = $stmt->fetchAll(PDO::FETCH_COLUMN);
             $patterns = array('', '/references', '/notes', '/images', '/interlocks', '/giving');
             foreach ($entityUrls as $u) {
                 foreach ($patterns as $p) {
                     $urls[] = $u . $p;
                 }
             }
             break;
         case 'relationship':
             //get relationship urls
             $q = LsDoctrineQuery::create()->select('r.id')->from('Relationship r')->where('r.is_deleted = 0');
             $stmt = $db->execute($q->getSqlQuery());
             $relIds = $stmt->fetchAll(PDO::FETCH_COLUMN);
             $patterns = array('view', 'references', 'notes');
             foreach ($relIds as $id) {
                 foreach ($patterns as $p) {
                     $urls[] = '/relationship/' . $p . '/id/' . $id;
                 }
             }
             break;
         case 'list':
             //get list urls
             $q = LsDoctrineQuery::create()->select("CONCAT('/', l.id, '/', REPLACE(REPLACE(l.name, ' ', '_'), '/', '~'))")->from('LsList l')->where('l.is_deleted = 0');
             $stmt = $db->execute($q->getSqlQuery());
             $listUrls = $stmt->fetchAll(PDO::FETCH_COLUMN);
             $patterns = array('', '/references', '/notes', '/members', '/interlocks', '/giving', '/funding');
             foreach ($listUrls as $u) {
                 foreach ($patterns as $p) {
                     $urls[] = '/list' . $u . $p;
                 }
             }
             break;
         case 'user':
             //get user urls
             $q = LsDoctrineQuery::create()->select('p.public_name')->from('sfGuardUserProfile p')->where('p.is_visible = 1 AND p.is_confirmed = 1')->setHydrationMode(Doctrine::HYDRATE_NONE);
             $stmt = $db->execute($q->getSqlQuery());
             $names = $stmt->fetchAll(PDO::FETCH_COLUMN);
             $patterns = array('', '/groups');
             foreach ($names as $name) {
                 foreach ($patterns as $p) {
                     $urls[] = '/user/' . $name . $p;
                 }
             }
             break;
         case 'group':
             //get group urls
             $q = LsDoctrineQuery::create()->select('g.name')->from('sfGuardGroup g')->where('g.is_working = 1 AND g.is_private = 0')->setHydrationMode(Doctrine::HYDRATE_NONE);
             $stmt = $db->execute($q->getSqlQuery());
             $names = $stmt->fetchAll(PDO::FETCH_COLUMN);
             foreach ($names as $name) {
                 $urls[] = '/group/' . $name;
             }
             break;
     }
     //break urls into sets of 50,000
     $num = ceil(count($urls) / 50000);
     $urlSets = array();
     for ($n = 0; $n < $num; $n++) {
         $urlSets[] = array_slice($urls, $n * 50000, 50000);
     }
     print "Found " . count($urls) . " " . $options['type'] . " URLs; writing to " . count($urlSets) . " Sitemap files...\n";
     //generate sitemap XML file for each set
     foreach ($urlSets as $key => $set) {
         $xml = '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
         $xml .= '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">' . "\n";
         foreach ($set as $url) {
             $xml .= "  <url>\n";
             $xml .= "    <loc>" . htmlspecialchars($this->urlBase . $url) . "</loc>\n";
             $xml .= "  </url>\n";
         }
         $xml .= "</urlset>";
         //write to file in /web
         $fileName = $options['type'] . '-sitemap.' . ($key + 1) . '.xml';
         $f = fopen(sfConfig::get('sf_web_dir') . '/' . $fileName, 'w');
         fwrite($f, $xml);
         fclose($f);
         $this->fileNames[] = $fileName;
         print "+ Wrote " . count($set) . " URLs to sitemap file: " . $fileName . "\n";
     }
     //DONE
     LsCli::beep();
 }