protected function execute($arguments = array(), $options = array())
 {
     // initialize database manager
     $databaseManager = new sfDatabaseManager($this->configuration);
     $databaseManager->initialize($this->configuration);
     $proxy_scraper = new ProxyScraper($options['test_mode'], $options['debug_mode'], $this->configuration);
     $proxy_scraper->setCorpIds($options['limit'], $options['start_id'], $options['ticker']);
     $proxy_scraper->run();
 }
 public function execute()
 {
     if (!$this->safeToRun('sec')) {
         $this->printDebug('script already running');
         die;
     }
     if (!isset($this->corp_ids)) {
         return null;
     }
     foreach ($this->corp_ids as $corp_id) {
         if (!$this->override && $this->hasMeta($corp_id, 'is_complete') && $this->getMeta($corp_id, 'is_complete')) {
             $this->printDebug("Already fetched roster for Entity " . $corp_id . "; skipping...");
             continue;
         } else {
             if (!$this->override && $this->hasMeta($corp_id, 'lacks_cik') && $this->getMeta($corp_id, 'lacks_cik')) {
                 $this->printDebug("No SEC cik found for Entity " . $corp_id . "; skipping...");
                 continue;
             }
         }
         try {
             echo number_format(memory_get_usage()) . "\n";
             $this->browser->restart($this->defaultHeaders);
             $this->db->beginTransaction();
             $corp = Doctrine::getTable('Entity')->find($corp_id);
             echo "\n*****************\n\nfetching roster for " . $corp->name . " (" . $corp->ticker . ")" . "\n\n";
             //grab the corporation's cik if it doesn't have one already
             if (!$corp->sec_cik) {
                 if ($result = $this->getCik($corp->ticker)) {
                     $corp->sec_cik = $result['cik'];
                     if ($corp->Industry->count() == 0) {
                         if ($result['sic']['name'] && $result['sic']['name'] != '') {
                             if (!($industry = LsDoctrineQuery::create()->from('Industry i')->where('i.name = ? and i.code = ?', array($result['sic']['name'], $result['sic']['code']))->fetchOne())) {
                                 $industry = new Industry();
                                 $industry->name = LsLanguage::nameize(LsHtml::replaceEntities($result['sic']['name']));
                                 $industry->context = 'SIC';
                                 $industry->code = $result['sic']['code'];
                                 $industry->save();
                                 $this->printDebug('Industry: ' . $industry->name . ' (' . $industry->code . ')');
                             }
                             $q = LsQuery::getByModelAndFieldsQuery('BusinessIndustry', array('industry_id' => $industry->id, 'business_id' => $corp->id));
                             if (!$q->fetchOne()) {
                                 $corp->Industry[] = $industry;
                             }
                         }
                         $corp->save();
                         $corp->addReference($result['url'], null, $corp->getAllModifiedFields(), 'SEC EDGAR Page');
                     }
                 } else {
                     $this->saveMeta($corp->id, 'lacks_cik', true);
                     $this->db->commit();
                     continue;
                 }
             }
             if ($corp->sec_cik) {
                 $form4_urls = $this->getForm4Urls($corp->sec_cik);
                 $roster = array();
                 foreach ($form4_urls as $url_arr) {
                     $result = $this->getForm4Data($url_arr, $corp->sec_cik);
                     if ($result) {
                         $roster[] = $result;
                     }
                 }
                 $proxy_urls = $this->getProxyUrls($corp->sec_cik, array('2007', '2008'));
                 if (count($proxy_urls)) {
                     $proxy_url = $proxy_urls[0]['url'];
                     $proxy_year = $proxy_urls[0]['year'];
                     //search proxy for names appearing on form 4s
                     $roster = $this->getProxyData($roster, $proxy_url, $proxy_year);
                 } else {
                     $this->saveMeta($corp->id, 'lacks_cik', true);
                     $this->db->commit();
                     continue;
                 }
                 $corp->addReference($proxy_url, null, null, $proxy_year . ' Proxy');
                 //loop through names found on form 4s and search proxy
                 foreach ($roster as $r) {
                     echo "\n" . $r['personName'] . " is director? " . $r['isDirector'] . " at " . $r['form4Url'] . " \n";
                     if (isset($r['proxyName'])) {
                         echo "in proxy as " . $r['proxyName'] . " \n";
                     } else {
                         echo "not in proxy \n\n";
                     }
                     //make sure this appears in the proxy and has either an officer title or is a director
                     if (isset($r['proxyName']) && ($r['isDirector'] == '1' || $r['officerTitle'] != '')) {
                         $p = EntityTable::getByExtensionQuery('BusinessPerson')->addWhere('businessperson.sec_cik = ?', $r['personCik'])->fetchOne();
                         if (!$p) {
                             $p = $this->importPerson($r, $corp->name);
                         }
                         if ($p) {
                             $this->importAddress($r['address'], $p, $r, $corp->name);
                             if ($r['isDirector'] == 1) {
                                 $this->importRelationship($p, $corp, 'Director', $r);
                             }
                             if ($r['officerTitle'] != '') {
                                 $descriptions = $this->parseDescriptionStr($r['officerTitle'], $corp);
                                 foreach ($descriptions as $d) {
                                     if ($d['note']) {
                                         $position = $d['description'] . ' (' . implode(', ', $d['note']) . ')';
                                     } else {
                                         $position = $d['description'];
                                     }
                                     $this->importRelationship($p, $corp, $position, $r);
                                 }
                             }
                         }
                     }
                 }
             }
             if (!$this->testMode) {
                 $this->db->commit();
             }
             if (isset($proxy_url)) {
                 $proxy_scraper = new ProxyScraper($this->testMode, $this->debugMode, $this->appConfiguration);
                 $proxy_scraper->setCorpIds(1, $corp->id);
                 $proxy_scraper->setProxy($this->proxyText, $proxy_url, $proxy_year);
                 $proxy_scraper->disableBeep();
                 $proxy_scraper->run();
             }
         } catch (Exception $e) {
             //something bad happened, rollback
             $this->db->rollback();
             throw $e;
         }
         $this->saveMeta($corp_id, 'is_complete', true);
     }
 }