protected function execute($arguments = array(), $options = array())
{
// initialize database manager
$databaseManager = new sfDatabaseManager($this->configuration);
$databaseManager->initialize($this->configuration);
$proxy_scraper = new ProxyScraper($options['test_mode'], $options['debug_mode'], $this->configuration);
$proxy_scraper->setCorpIds($options['limit'], $options['start_id'], $options['ticker']);
$proxy_scraper->run();
}
public function execute()
{
if (!$this->safeToRun('sec')) {
$this->printDebug('script already running');
die;
}
if (!isset($this->corp_ids)) {
return null;
}
foreach ($this->corp_ids as $corp_id) {
if (!$this->override && $this->hasMeta($corp_id, 'is_complete') && $this->getMeta($corp_id, 'is_complete')) {
$this->printDebug("Already fetched roster for Entity " . $corp_id . "; skipping...");
continue;
} else {
if (!$this->override && $this->hasMeta($corp_id, 'lacks_cik') && $this->getMeta($corp_id, 'lacks_cik')) {
$this->printDebug("No SEC cik found for Entity " . $corp_id . "; skipping...");
continue;
}
}
try {
echo number_format(memory_get_usage()) . "\n";
$this->browser->restart($this->defaultHeaders);
$this->db->beginTransaction();
$corp = Doctrine::getTable('Entity')->find($corp_id);
echo "\n*****************\n\nfetching roster for " . $corp->name . " (" . $corp->ticker . ")" . "\n\n";
//grab the corporation's cik if it doesn't have one already
if (!$corp->sec_cik) {
if ($result = $this->getCik($corp->ticker)) {
$corp->sec_cik = $result['cik'];
if ($corp->Industry->count() == 0) {
if ($result['sic']['name'] && $result['sic']['name'] != '') {
if (!($industry = LsDoctrineQuery::create()->from('Industry i')->where('i.name = ? and i.code = ?', array($result['sic']['name'], $result['sic']['code']))->fetchOne())) {
$industry = new Industry();
$industry->name = LsLanguage::nameize(LsHtml::replaceEntities($result['sic']['name']));
$industry->context = 'SIC';
$industry->code = $result['sic']['code'];
$industry->save();
$this->printDebug('Industry: ' . $industry->name . ' (' . $industry->code . ')');
}
$q = LsQuery::getByModelAndFieldsQuery('BusinessIndustry', array('industry_id' => $industry->id, 'business_id' => $corp->id));
if (!$q->fetchOne()) {
$corp->Industry[] = $industry;
}
}
$corp->save();
$corp->addReference($result['url'], null, $corp->getAllModifiedFields(), 'SEC EDGAR Page');
}
} else {
$this->saveMeta($corp->id, 'lacks_cik', true);
$this->db->commit();
continue;
}
}
if ($corp->sec_cik) {
$form4_urls = $this->getForm4Urls($corp->sec_cik);
$roster = array();
foreach ($form4_urls as $url_arr) {
$result = $this->getForm4Data($url_arr, $corp->sec_cik);
if ($result) {
$roster[] = $result;
}
}
$proxy_urls = $this->getProxyUrls($corp->sec_cik, array('2007', '2008'));
if (count($proxy_urls)) {
$proxy_url = $proxy_urls[0]['url'];
$proxy_year = $proxy_urls[0]['year'];
//search proxy for names appearing on form 4s
$roster = $this->getProxyData($roster, $proxy_url, $proxy_year);
} else {
$this->saveMeta($corp->id, 'lacks_cik', true);
$this->db->commit();
continue;
}
$corp->addReference($proxy_url, null, null, $proxy_year . ' Proxy');
//loop through names found on form 4s and search proxy
foreach ($roster as $r) {
echo "\n" . $r['personName'] . " is director? " . $r['isDirector'] . " at " . $r['form4Url'] . " \n";
if (isset($r['proxyName'])) {
echo "in proxy as " . $r['proxyName'] . " \n";
} else {
echo "not in proxy \n\n";
}
//make sure this appears in the proxy and has either an officer title or is a director
if (isset($r['proxyName']) && ($r['isDirector'] == '1' || $r['officerTitle'] != '')) {
$p = EntityTable::getByExtensionQuery('BusinessPerson')->addWhere('businessperson.sec_cik = ?', $r['personCik'])->fetchOne();
if (!$p) {
$p = $this->importPerson($r, $corp->name);
}
if ($p) {
$this->importAddress($r['address'], $p, $r, $corp->name);
if ($r['isDirector'] == 1) {
$this->importRelationship($p, $corp, 'Director', $r);
}
if ($r['officerTitle'] != '') {
$descriptions = $this->parseDescriptionStr($r['officerTitle'], $corp);
foreach ($descriptions as $d) {
if ($d['note']) {
$position = $d['description'] . ' (' . implode(', ', $d['note']) . ')';
} else {
$position = $d['description'];
}
$this->importRelationship($p, $corp, $position, $r);
}
}
}
}
}
}
if (!$this->testMode) {
$this->db->commit();
}
if (isset($proxy_url)) {
$proxy_scraper = new ProxyScraper($this->testMode, $this->debugMode, $this->appConfiguration);
$proxy_scraper->setCorpIds(1, $corp->id);
$proxy_scraper->setProxy($this->proxyText, $proxy_url, $proxy_year);
$proxy_scraper->disableBeep();
$proxy_scraper->run();
}
} catch (Exception $e) {
//something bad happened, rollback
$this->db->rollback();
throw $e;
}
$this->saveMeta($corp_id, 'is_complete', true);
}
}