protected function importGovernor($row) { $url = $this->_baseUrl . $row['url']; if (!$this->browser->get($url)->responseIsError()) { $text = $this->browser->getResponseText(); $text = LsHtml::replaceEntities($text); //preg_match('/>Family\:<\/b>([^<]*)<br/is',$text,$family_arr); $name = trim(str_ireplace('Gov.', '', $row['name'])); $this->printDebug(''); $this->printDebug($name . ':'); $governor = PersonTable::parseFlatName($name); $governor->addExtension('PoliticalCandidate'); $governor->addExtension('ElectedRepresentative'); $governor->is_state = 1; $similar = $governor->getSimilarEntitiesQuery(true)->execute(); foreach ($similar as $s) { $sim_re = LsString::escapeStringForRegex($s->name_first); $search_re = LsString::escapeStringForRegex($governor->name_first); if (preg_match('/^' . $sim_re . '/su', $governor->name_first) == 0 && preg_match('/^' . $search_re . '/su', $s->name_first) == 0) { continue; } $bio = $s->getExtendedBio(); if (preg_match('/\\bgovernor(ship)?\\b/isu', $bio)) { $governor = $s; $this->printDebug(' Found existing governor: ' . $s->name . ' ' . $s->id); break; } } $governor->save(); $this->printDebug($governor->id); if (!$governor->start_date && preg_match('/>Born\\:<\\/b>([^<]*)<br/is', $text, $birth_arr)) { $this->printDebug(' Birthdate: ' . $birth_arr[1]); $governor->start_date = trim($birth_arr[1]); } if (!$governor->birthplace && preg_match('/>Birth State\\:<\\/b>([^<]*)<br/is', $text, $birth_state_arr)) { $this->printDebug(' Birthplace: ' . trim($birth_state_arr[1])); $governor->birthplace = trim($birth_state_arr[1]); } //PARTY MEMBERSHIP if (preg_match('/>Party\\:<\\/b>([^<]*)<br/is', $text, $party_arr)) { $party_str = $party_arr[1]; $this->printDebug(' Party: ' . $party_str); if (stristr($party_str, 'Democrat')) { $party = EntityTable::getByExtensionQuery('PoliticalParty')->addWhere('name = ?', 'Democratic Party')->fetchOne(); } if (stristr($party_str, 'Republican')) { $party = EntityTable::getByExtensionQuery('PoliticalParty')->addWhere('name = ?', 'Republican Party')->fetchOne(); } if (isset($party) && $party && !$governor->party_id) { $governor->Party = $party; $governor->is_independent = false; $this->printDebug(' Added membership in ' . $party); } else { if (stristr($party_str, 'Independent')) { $governor->is_independent = true; } } } if (!$governor->summary && preg_match_all('/>([^<]{240,})/isu', $text, $bio_match)) { $str = ''; foreach ($bio_match[1] as $b) { if (!stristr($b, 'Javascript')) { $str .= "\n\n" . $b; } } $str = trim($str); if (strlen($str)) { $governor->summary = $str; } } $governor->save(); $governor->addReference($url, null, $governor->getAllModifiedFields(), 'Governors Association'); //SCHOOLS if (preg_match('/>School\\(s\\)\\:<\\/b>([^<]*)<br/is', $text, $school_arr)) { $school_names = explode(';', trim($school_arr[1])); if (count($school_names) == 1) { $school_names = explode(',', $school_names[0]); } foreach ($school_names as $school_name) { $school_name = trim($school_name); if (!($school = EntityTable::getByExtensionQuery('School')->leftJoin('e.Alias a')->addWhere('e.name = ? or a.name = ?', array($school_name, $school_name))->fetchOne())) { $school = new Entity(); $school->addExtension('Org'); $school->addExtension('School'); $school->name = $school_name; $school->save(); $this->printDebug(' Added School: ' . $school_name); } $q = RelationshipTable::getByCategoryQuery('Education')->addWhere('entity1_id = ? and entity2_id = ?', array($governor->id, $school->id))->fetchOne(); if (!$q) { $relationship = new Relationship(); $relationship->setCategory('Education'); $relationship->Entity1 = $governor; $relationship->Entity2 = $school; $relationship->is_current = 0; $relationship->save(); $relationship->addReference($url, null, $relationship->getAllModifiedFields(), 'Governors Association'); $this->printDebug(' Added education: ' . $relationship->name); } } } //GOVERNOR OFFICE AND POSITION $office_name = 'Office of the Governor of ' . $row['state']; if (!($office = EntityTable::getByExtensionQuery('GovernmentBody')->addWhere('name = ?', $office_name)->fetchOne())) { $office = new Entity(); $office->name = $office_name; $office->addExtension('Org'); $office->addExtension('GovernmentBody'); $state = Doctrine::getTable('AddressState')->findOneByName($row['state']); if ($state) { $office->state_id = $state->id; } $office->save(); $office->addReference($url, null, $office->getAllModifiedFields(), 'Governors Association'); $this->printDebug(' Added office: ' . $office->name); } $q = RelationshipTable::getByCategoryQuery('Position')->addWhere('entity1_id = ? and entity2_id = ? and description1 = ?', array($governor->id, $office->id, 'Governor'))->fetchOne(); if (!$q) { sort($row['years']); $i = 0; while ($i < count($row['years'])) { $governorship = new Relationship(); $governorship->setCategory('Position'); $governorship->Entity1 = $governor; $governorship->Entity2 = $office; $governorship->description1 = 'Governor'; $governorship->start_date = $row['years'][$i]; $i++; if (isset($row['years'][$i])) { $governorship->end_date = $row['years'][$i]; $governorship->is_current = 0; if (!$governor->blurb && !isset($row['years'][$i + 1])) { $governor->blurb = 'Former Governor of ' . $row['state']; } } else { $governorship->is_current = 1; if (!$governor->blurb) { $governor->blurb = 'Governor of ' . $row['state']; } } $governor->save(); $i++; $governorship->save(); $governorship->addReference($url, null, $governorship->getAllModifiedFields(), 'Governors Association'); $this->printDebug(' Added governorship: ' . $governorship->name); } } //SPOUSE if (preg_match('/>Spouse\\:<\\/b>(.*?)<br/is', $text, $spouse_arr)) { $spouse = trim(LsHtml::stripTags($spouse_arr[1])); $q = RelationshipTable::getByCategoryQuery('Family')->addWhere('entity1_id = ? or entity2_id = ?', array($governor->id, $governor->id))->fetchOne(); if (!$q && strlen($spouse)) { $spouse = PersonTable::parseFlatName($spouse); $spouse->save(); $this->printDebug(' Added spouse: ' . $spouse->name); $relationship = new Relationship(); $relationship->setCategory('Family'); $relationship->Entity1 = $spouse; $relationship->Entity2 = $governor; $relationship->description1 = 'Spouse'; $relationship->description2 = 'Spouse'; $relationship->save(); $relationship->addReference($url, null, $relationship->getAllModifiedFields(), 'Governors Association'); $this->printDebug(' Added spouse relationship: ' . $relationship->name); } } //ADDRESS --not working, malformed addresses /* if (preg_match('/>Address\:\s*<\/b>(.*?)<b>/is',$text,$address_arr)) { $address = trim(str_replace('<br/>',', ',$address_arr[1])); $this->printDebug($address); if ($governor->Address->count() == 0 && $a = $governor->addAddress($address)) { $this->printDebug(' Address: ' . $a); $governor->save(); } }*/ //PHONE NUMBER if (preg_match('/>Phone\\(s\\)\\:<\\/b>([^<]*)<br/is', $text, $phone_arr)) { $phone_number = trim($phone_arr[1]); if (!$governor->Phone->count()) { $phone = $governor->addPhone($phone_number); $this->printDebug(' Phone: ' . $phone); } } if (!$governor->Image->count() && preg_match('/<img .*?class\\="display" src\\="([^"]*)"/is', $text, $img_arr)) { $url = $img_arr[1]; try { $fileName = ImageTable::createFiles($url, $governor->name_first); } catch (Exception $e) { $fileName = null; } if ($fileName) { //insert image record $image = new Image(); $image->filename = $fileName; $image->entity_id = $governor->id; $image->title = $governor->name; $image->caption = 'From Governors Association website'; $image->is_featured = true; $image->is_free = false; $image->url = $url; $image->save(); $this->printDebug("Imported image: " . $image->filename); } } } }
private function findPersonBio($page, $person, $org) { //$this->printDebug(''); $name_re = LsString::escapeStringForRegex($person->name_last); if (preg_match('/<title>([^<]*)<\\/title>/is', $page, $match)) { if (stristr($match[1], $person->name_last) && stristr($match[1], $person->name_first) && strlen($person->name_first) > 2) { $name_re .= '|' . LsString::escapeStringForRegex($person->name_first); } } $layout_tags = implode('|', LsHtml::$layoutTags); $re2 = '/>([^<]*?(' . $name_re . ')(\\s|,|<)(.*?))<\\/?(' . $layout_tags . ')/is'; $re = $re2 . 'u'; //$this->printDebug($re); $bio_match = null; if (preg_match_all($re, $page, $matches) || preg_match_all($re2, $page, $matches)) { //$this->printDebug('matches found'); $arr = array(); $most_reqs = 0; $qual = false; $news = false; foreach ($matches[1] as $match) { if (stristr($match, '}') || stristr($match, '{') || preg_match('/\\svar\\s/is', $match)) { //$this->printDebug('FAILED - curly brackets'); continue; } $str = LsHtml::replaceEntities($match); $str = LsHtml::stripTags($str, ''); $str = trim(LsString::spacesToSpace($str)); $this->printDebug(strlen($str)); if (strlen($str) > 3000) { $this->printDebug('FAILED - str too long'); continue; } if (preg_match('/(^|\\b)(' . $name_re . ')\\b/is', $str) == 0) { $this->printDebug($match . 'FAILED - no name match'); continue; } $word_count = count(explode(' ', $str)); if ($word_count < 12) { $this->printDebug('FAILED - str not long enough'); continue; } else { if (stristr($str, 'announce') || stristr($str, 'today') || stristr($str, '—') || stristr($str, '–') || preg_match('/^[^\\-]{0,100}\\-(\\-|\\s)/is', $str)) { $news = true; $this->printDebug('FAILED: dash / announced / today'); } else { if (preg_match('/(^|\\s)([\'"”])([^\\1]+)\\1/is', $str, $qm) && count(explode(' ', $qm[0])) > 6) { $news = true; $this->printDebug('FAILED: quote'); } else { if (preg_match_all('/\\s(\\p{Ll})+\\b/su', $str, $lcm) < 5) { $this->printDebug('FAILED: not enough lowercase'); } else { $bio_words = PersonTable::$commonBioWords; if (in_array('Lobbyist', $person->getExtensions())) { $bio_words = array_merge($bio_words, LobbyistTable::$commonBioWords); } $bio_words = implode('|', $bio_words); $bio_word_ct = preg_match_all('/\\s(' . $bio_words . ')\\s/is', $str, $matches); $str = trim($str); if (preg_match('/\\.$/is', $str) == 0) { $this->printDebug('no period at end of string'); } else { if ($bio_word_ct > 1) { $news = false; $qual = true; $arr[] = $str; } else { $this->printDebug('less than 2 bio words'); if ($news == false) { $str = preg_replace('/^[\\,\\.\\:\\;]\\s*/su', '', $str); $arr[] = $str; //array('str' => $str, 'bio_words' => $bio_word_ct); } } } } } } //$this->printDebug(''); } } if ($qual) { $arr = array_unique($arr); $ret = false; $bio = implode("\n\n", $arr); //$this->printDebug($name_re); if (strlen($bio) < 3000 && LsString::withinN($bio, '(' . $name_re . ')', '(is|was|holds|led|has|had|provides|practices|served|leads)', 2)) { if (preg_match('/^.*?\\b(' . $name_re . ')\\b/is', $bio, $m) && count(explode(' ', $m[0])) < 20) { $ret = true; $this->printDebug('SUCCESS'); } } else { $this->printDebug('within N failed !!!!'); } $org_test = true; if ($ret && stristr($org->name, $person->name_last)) { $org_test = false; if (strlen($person->name_first) > 1) { if (preg_match('/([^\\s]+\\s+){0,14}/is', $arr[0], $beg_match)) { $nf_re = LsString::escapeStringForRegex($person->name_first); if (preg_match('/\\b' . $nf_re . '\\b/is', $beg_match[0]) || preg_match('/\\b(Mr|Mrs|Ms)\\b/su', $arr[0])) { $org_test = true; //$this->printDebug('PASSED FIRST NAME TEST'); } } } else { if (preg_match('/\\b(he|she|him|her|his|mr|ms|mrs)\\b/is', $arr[0])) { $org_test = true; //$this->printDebug('PASSED POSSESSIVE TEST'); } } } if ($ret && $org_test) { return $bio; } } } else { $this->printDebug('no matches found'); } return false; }
public function __construct($row) { $this->id = $row[1]; $this->name = trim(LsHtml::replaceEntities($row[2])); $this->lifespan = trim(LsHtml::replaceEntities($row[3])); $this->type = trim($row[4]); $this->party = trim($row[5]); $this->state = trim($row[6]); list($this->termStart, $this->termEnd) = explode('-', trim($row[7])); }
public function getSummary($str, Entity $e) { $str = LsHtml::replaceEntities($str); $name_re = array(); $name_re[] = $e->getNameRegex(); if ($e->name_nick && $e->name_nick != '') { $name_re[] = LsString::escapeStringForRegex($e->name_nick); } $name_re = implode('|', $name_re); $style_tags = implode('|', LsHtml::$fontStyleTags); $layout_tags = implode('|', LsHtml::$layoutTags); $re = '/((' . $name_re . ')(.*?))<\\/?(' . $layout_tags . ')/isu'; $this->printDebug($re); $results = null; if (preg_match_all($re, $str, $matches)) { $results = $matches[1]; foreach ($results as $result) { $result = LsString::spacesToSpace(LsHtml::stripTags($result)); $this->printDebug($result); } } return $results; }
private function getProxy() { $text = null; $this->printDebug($this->corp->name); $url = "http://searchwww.sec.gov/EDGARFSClient/jsp/EDGAR_Query_Result.jsp?startDoc=1&queryString=&queryForm=DEF+14A&isAdv=1&queryCik=" . $this->corp->sec_cik . "&numResults=10#topAnchor"; $years = implode('|', $this->years); if ($this->browser->get($url)->responseIsError()) { echo "Couldn't get " . $url . "\n"; return; } $re = '/(' . $years . ')<\\/i>(<[^>]*>){2}<a[^\']+\'([^\']+)(?<=\\.htm)\'[^>]*>([^<]*)</isu'; $text = $this->browser->getResponseText(); //echo $text; $matched = preg_match_all($re, $text, $matches, PREG_SET_ORDER); if ($matched > 0) { foreach ($matches as $match) { if (stristr($match[3], 'def14') !== false || stristr($match[4], 'def 14') !== false) { $this->year = $match[1]; //$this->printDebug($this->year); $this->url = $match[3]; break; } } if ($this->browser->get($this->url)->responseIsError()) { echo "Couldn't get " . $this->url . "\n"; return; } $this->printDebug($this->url); $text = $this->browser->getResponseText(); $text = LsHtml::replaceEntities($text); $text = LsString::utf8TransUnaccent($text); $this->text = $text; } }
public function parseResults($match) { if (isset($match['bio'])) { $bio_dirty = LsHtml::replaceEntities(LsString::spacesToSpace(LsHtml::stripTags($match['bio'], "; "))); $bio_dirty = preg_replace('/(\\;\\s)+/is', '; ', $bio_dirty); } foreach ($match as $k => &$m) { $m = LsHtml::replaceEntities(LsString::spacesToSpace(LsHtml::stripTags($m, " "))); } if (isset($match['name'])) { $name = $match['name']; $bio = ''; if (isset($match['bio'])) { $bio = $match['bio']; } } else { return; } $this->printDebug("_________________________\n\nname: " . $name . "\n"); $this->printDebug("bio: " . $bio . "\n"); $accept = strtolower($this->readline('Process this entity? (n to skip) ')); if ($accept == 'n' || $accept == 'no') { return false; } if (!$this->org_org) { if ($this->last_first) { $entity = PersonTable::parseCommaName($name); } else { $entity = PersonTable::parseFlatName($name); } $similar_entities = PersonTable::getSimilarQuery2($entity)->execute(); } else { $entity = new Entity(); $entity->addExtension('Org'); foreach ($this->org_extensions as $ext) { $entity->addExtension($ext); } $entity->setEntityField('name', $name); $name = trim($name); $name = str_replace('.', '', $name); $similar_entities = OrgTable::getSimilarQuery($entity)->execute(); } $matched = false; foreach ($similar_entities as $similar_entity) { if ($similar_entity['primary_ext'] == 'Person') { $this->printDebug(' POSSIBLE MATCH: ' . $similar_entity->name . ' (Orgs :: ' . $similar_entity->getRelatedOrgsSummary() . " Bio :: {$similar_entity->summary})"); } else { $this->printDebug(' POSSIBLE MATCH: ' . $similar_entity->name . ' (Summary :: ' . $similar_entity->summary . ')'); } $accept = $this->readline(' Is this the same entity? (y or n)'); $attempts = 1; while ($accept != 'y' && $accept != 'n' && $attempts < 5) { $accept = $this->readline(' Is this the same entity? (y or n) '); $attempts++; } if ($accept == 'y') { $entity = $similar_entity; $matched = true; $this->printDebug(' [accepted]'); //sleep(1); break; } else { if ($accept == 'break') { break; } } } $created = false; if (!$matched) { if ($entity->getPrimaryExtension() == 'Person') { $this->printDebug(' New person: ' . $entity->name_first . ' ' . $entity->name_last); } else { $this->printDebug(' New org: ' . $entity->name); } $accept = $this->readline(' create this new entity? (y or n) '); $attempts = 1; while ($accept != 'y' && $accept != 'n' && $attempts < 5) { $accept = $this->readline(' create this new entity? (y or n) '); $attempts++; } if ($accept == 'y') { if ($entity->getPrimaryExtension() == 'Person') { $this->printDebug("\n Bio: {$bio} \n"); $accept = $this->readline(' Add this bio? (y or n) '); $attempts = 1; while ($accept != 'y' && $accept != 'n' && $attempts < 5) { $accept = $this->readline(' add this bio? (y or n) '); $attempts++; } if ($accept == 'y') { $entity->summary = $bio; } } $entity->save(); $entity->addReference($this->url, null, null, $this->url_name); $created = true; $this->printDebug(' ' . $entity->name . ' saved'); //sleep(1); } } if (($matched || $created) && $entity->getPrimaryExtension() == 'Person') { $accept = $this->readline("Parse above bio for possible relationships? (y or n) "); $attempts = 1; while ($accept != 'y' && $accept != 'n' && $attempts < 5) { $accept = $this->readline("Parse above bio for possible relationships? (y or n) "); $attempts++; } if ($accept == 'y') { $names = $entity->parseBio($bio_dirty); $this->printDebug(" Orgs that {$entity} has a position at?"); foreach ($names as $name) { $exists = false; $name = trim($name); $accept = $this->readline(" > {$name} :: an org? (y or n or b to break) "); $attempts = 1; $accept = strtolower($accept); while ($accept != 'y' && $accept != 'n' && $accept != 'b' && $attempts < 5) { $accept = $this->readline(" {$name} :: an org? (y or n or b to break) "); $accept = strtolower($accept); $attempts++; } if ($accept == 'b') { break; } else { if ($accept == 'y') { $this->printDebug(' .....looking for names.....'); $orgs = EntityTable::getByExtensionAndNameQuery('Org', $name)->limit(10)->execute(); $related_org = null; foreach ($orgs as $org) { $q = LsDoctrineQuery::create()->from('Relationship r')->where('entity1_id = ? and entity2_id = ?', array($entity->id, $org->id))->fetchOne(); if ($q) { $this->printDebug(' Position already exists, skipping...'); $exists = true; break; } $accept = $this->readline(" Create a position relationship between {$entity->name} and {$org->name}? (y or n) "); $attempts = 1; while ($accept != 'y' && $accept != 'n' && $attempts < 5) { $accept = $this->readline(" Create a position relationship between {$entity->name} and {$org->name}? (y or n) "); $attempts++; } if ($accept == 'y') { $related_org = $org; break; } } if (!$related_org && !$exists) { $accept = $this->readline(" couldn't find org, should this one be created: {$name} (y or n) "); while ($accept != 'y' && $accept != 'n' && $attempts < 5) { $accept = $this->readline(" couldn't find org, should this one be created: {$name} (y or n) "); $attempts++; } if ($accept == 'y') { $related_org = new Entity(); $related_org->addExtension('Org'); $related_org->name = preg_replace('/\\.(?!com)/i', '', $name); $extensions = $this->readline(" what extensions should this org get? (eg 'Business, LobbyingFirm, LawFirm') "); $extensions = preg_split('/\\,\\s*/isu', $extensions, -1, PREG_SPLIT_NO_EMPTY); try { foreach ($extensions as $extension) { $related_org->addExtension($extension); } $related_org->save(); $related_org->addReference($this->url, null, null, $this->url_name); } catch (Exception $e) { $this->printDebug(' !!! problems with org creation, skipping'); $related_org = null; } } } if ($related_org) { $q = LsDoctrineQuery::create()->from('Relationship r')->where('r.entity1_id = ? and r.entity2_id = ? and r.category_id = ?', array($entity->id, $related_org->id, 1))->fetchOne(); if ($q) { $this->printDebug(' (relationship already found, skipping...)'); continue; } $relationship = new Relationship(); $relationship->Entity1 = $entity; $relationship->Entity2 = $related_org; $relationship->setCategory('Position'); $title = $this->readline(" Title for this position relationship? (<enter> to skip) "); if (strlen($title) > 2) { $relationship->description1 = $title; } $current = strtolower($this->readline(" Is the relationship current? (y or n or <enter> to skip) ")); if (in_array($current, array('y', 'yes'))) { $relationship->is_current = 1; } else { if (in_array($current, array('n', 'no'))) { $relationship->is_current = 0; } } $board = strtolower($this->readline(" Is the relationship a board position? (y or n or <enter> to skip) ")); if (in_array($board, array('y', 'yes'))) { $relationship->is_board = 1; } else { if (in_array($board, array('n', 'no'))) { $relationship->is_board = 0; } } $relationship->save(); $relationship->addReference($this->url, null, null, $this->url_name); $this->printDebug(" Relationship saved: {$relationship}"); } } } } } } if ($matched || $created) { if ($this->list) { $q = LsDoctrineQuery::create()->from('LsListEntity l')->where('l.entity_id = ? and l.list_id = ?', array($entity->id, $this->list->id))->fetchOne(); if (!$q) { $le = new LsListEntity(); $le->Entity = $entity; $le->LsList = $this->list; if (isset($match['rank'])) { if (preg_match('/(\\d+)/isu', $match['rank'], $m)) { $le->rank = $m[1]; } } $le->save(); $this->printDebug('List membership saved'); } } if ($this->org) { $q = LsDoctrineQuery::create()->from('Relationship r')->where('r.entity1_id = ? and r.entity2_id = ? and r.category_id = ?', array($entity->id, $this->org->id, 1))->fetchOne(); if ($q) { $this->printDebug(' (relationship already found, skipping...)'); return; } $relationship = new Relationship(); $relationship->Entity1 = $entity; $relationship->Entity2 = $this->org; $relationship->setCategory($this->relationship_category); if ($this->description1) { $relationship->description1 = $this->description1; } else { $description = $this->readline(" what description to give this relationship ({$relationship}) ? (less than 3 chars will skip)"); if (strlen($description) > 2) { $relationship->description1 = $description; } } if ($this->relationship_category == 'Position') { $relationship->is_board = $this->is_board; } else { if ($this->relationship_category == 'Donation') { if ($this->amount) { $relationship->amount = $this->amount; } else { $amount = $this->readline(" what amount ({$relationship}) ? (less than 3 chars will skip)"); if (strlen($amount) > 1) { $relationship->amount = $amount; } } } } $relationship->save(); $relationship->addReference($this->url, null, null, $this->url_name); $this->printDebug(" Relationship saved: {$relationship}"); } } //dump history if (isset($match['affiliation1'])) { $affiliation = $match['affiliation']; //$this->printDebug($affiliation); } }
function __construct($text) { $text = LsHtml::replaceEntities($text); $text = LsString::utf8TransUnaccent($text); $this->text = $text; }
static function getHtmlPersonNames($text) { $name_matches = array(); $re = '/>\\s*\\p{Lu}\'?(\\p{L}+|\\.)?\\s+\\p{Lu}\\.?\\s+\\p{Lu}\\p{L}+(\\,?\\s+\\p{Lu}\\p{L}{1,4}\\.?)?/su'; $re2 = '/>\\s*(\\p{Lu}\'?(\\p{L}+|\\.)?\\s+(\\p{Lu}\'?(\\s+|\\p{L}+\\s+|\\.\\s*)?){0,2}\\p{Lu}\'?\\p{L}+(\\-\\p{Lu}\'?\\p{L}+)?(\\,?\\s+\\p{Lu}\\p{L}{1,4}\\.?)?)\\**\\s*</su'; $re3 = '/>\\s*(\\p{Lu}\'?\\p{L}+(\\-\\p{Lu}\'?\\p{L}+)?\\,\\s+(\\p{Lu}\'?(\\p{L}+|\\.)?(\\s+(\\p{Lu}\'?(\\s+|\\p{L}+\\s+|\\.\\s*)?){0,2})?)(\\,?\\s+\\p{Lu}\\p{L}{1,4}\\.?)?)\\**\\s*</su'; $text = LsHtml::replaceEntities($text); $name_matches = array(); if (preg_match_all($re2, $text, $matches, PREG_OFFSET_CAPTURE)) { //LOOP THROUGH MATCHES TO CONFIRM NAMES for ($i = 0; $i < count($matches[1]); $i++) { $m = $matches[1][$i]; //echo $m[0] . "\n"; $is_name = false; if (preg_match('/\\s+\\p{Lu}\\.?\\s/', $m[0])) { //echo ' * initial' . "\n"; $is_name = true; } $parts = LsString::split(trim($m[0])); //ADD NAME TO MATCH LIST IF IT FITS CONDITIONS if (in_array($parts[0], LsLanguage::$commonFirstNames)) { //echo ' * first name' . "\n"; $is_name = true; } $q = LsDoctrineQuery::create()->from('Person p')->where('p.name_first = ?', $parts[0]); if ($q->count() > 0) { //echo ' LS name' . "\n"; $is_name = true; } if ($is_name) { $name_matches[] = $m[0]; } /* if ($i != 0) { $beg = $matches[1][$i-1][1]; $tweenstr = substr($text,$beg, $m[1] - $beg); //echo ' tag count: ' . LsHtml::tagCount($tweenstr) . "\n"; } preg_match('/^[^\s]+\s/su',trim($m[0]),$match); $tags = LsHtml::getSurroundingTags($text,$m[1],3);*/ } } if (preg_match_all($re3, $text, $matches, PREG_OFFSET_CAPTURE)) { for ($i = 0; $i < count($matches[1]); $i++) { $m = $matches[1][$i]; //echo $m[0] . "\n"; $person = PersonTable::parseCommaName($m[0]); $name_matches[] = $person->getFullName(false); } } return $name_matches; }
public function checkBoardPage($board_rels) { $goog = new LsGoogle(); $goog->setQuery($this->entity->name . " board"); $results = $goog->execute(); $results = $goog->parseSearchResults($results); if (count($results)) { $url = $results[0]['unescapedUrl']; if (stripos($url, "yahoo.com")) { $url = $results[1]['unescapedUrl']; } $this->printDebug($url); try { if (!$this->browser->get($url)->responseIsError()) { $text = $this->browser->getResponseText(); $text = LsHtml::replaceEntities($text); $status_arr = array(); $unique_arr = array(); foreach ($board_rels as $br) { $found = 0; //$this->printDebug($br->Entity1->getNameRegex()); $regexes = $br->Entity1->getNameRegexes(); foreach ($regexes as $regex) { if (preg_match_all($regex, $text, $matches, PREG_SET_ORDER | PREG_OFFSET_CAPTURE)) { //var_dump($matches); $found = 1; if (!in_array($br->entity1_id, $unique_arr)) { $unique_arr[] = $br->entity1_id; } break; } } $status_arr[] = $found; $this->printDebug($br->Entity1->name . " > " . $found); } if (count($unique_arr) > 1) { $this->printDebug("\tenough board member names found to mark as current or not"); for ($i = 0; $i < count($status_arr); $i++) { $br = $board_rels[$i]; if ($status_arr[$i] == 1) { $br->is_current = 1; } else { $br->is_current = 0; } if (!$this->testMode) { $br->save(); $br->addReference($url, null, null, $this->entity->name . ' board', null, null); } } return 1; } else { return 0; } } } catch (Exception $e) { return 0; } } else { return -1; } }
public function getCleanFirstParagraph() { if ($this->_paragraphs) { return null; } $first = $this->_paragraphs[0]; $first = LsString::spacesToSpace(LsHtml::replaceEntities(LsHtml::stripTags($first))); return $first; }
private function getProxyData($roster, $url, $proxy_year) { echo "fetching data from proxy at {$url} \n\n"; $people_count = 0; if (!$this->browser->get($url)->responseIsError()) { $this->proxyText = $this->browser->getResponseText(); $this->proxyText = LsHtml::replaceEntities($this->proxyText, ENT_QUOTES, 'UTF-8'); $this->proxyText = LsString::utf8TransUnaccent($this->proxyText); foreach ($roster as &$r) { //make sure this is not form 4 data for a corporation, continue to the next if it is if ($r['officerTitle'] == '' && $r['isDirector'] != 1 && strtoupper($r['isDirector']) != strtoupper('true')) { continue; } //echo $re; $parts = preg_split("/[\\s|\\.]+/", $r['personName'], -1, PREG_SPLIT_NO_EMPTY); //first word, but has to be part of last name because form4 names are in format RUBIN ROBERT E $last = trim($parts[0]); //sometimes O'LEARY can appear as O LEARY in the form 4 if (strlen($last) == 1) { $r['personName'] = $last . substr($r['personName'], 2); $parts = preg_split("/[\\s|\\.]+/", $r['personName'], -1, PREG_SPLIT_NO_EMPTY); $last = trim($parts[0]); } //prepare regex to match occurrences of full name //case insensitive to accommodate for various irregularities in names $re = LsLanguage::buildLooseNameRegex($r['personName']); $offset = 0; $found = true; //use stripos (much faster than regex) to find occurrences of the first word in the form 4 name (assumed to be part of the last name) //needs to be case insensitive //continue searching for last name in proxy until a matching full name (proxyName) is found while (!isset($r['proxyName']) && $found !== false) { $found = stripos($this->proxyText, $last, $offset); //$this->printDebug('found at pos:' . $found); $offset = $found + 1; if ($found !== false) { $str = substr($this->proxyText, $found - 70, 120); //$this->printDebug('found string: ' . $str); //$this->printDebug($re); preg_match_all($re, $str, $matches, PREG_OFFSET_CAPTURE | PREG_SET_ORDER); //$this->printDebug('matchcount is ' . count($matches)); foreach ($matches as $match) { if (stristr($match[1][0], '=')) { continue; } //since we may or may not be working with the full last name, use getLastName to return full last name $new_last = $this->getLastName($r['personName'], $match[1][0]); if ($new_last) { //if last name produced by case insensitive search has no capital letters, not a match if (preg_match('/\\p{Lu}/su', $new_last) == 0) { continue; } //now that we have a last name, pull the full name from the string $name = LsLanguage::getNameWithLast($match[0][0], $new_last); if ($name) { $parts = preg_split('/\\s+/isu', $name['nameStart'], -1, PREG_SPLIT_NO_EMPTY); $non_prefixes = array_diff($parts, PersonTable::$nameParsePrefixes); //if all we've found are matching prefixes, not a match if (count($non_prefixes) == 0) { continue; } else { $name1_parts = preg_split('/\\s+/', $r['personName'], -1, PREG_SPLIT_NO_EMPTY); $ct = 0; //compatibility check to correct for vagueness of regex foreach ($non_prefixes as $n) { foreach ($name1_parts as $p) { if (stripos($n, $p) === 0 || stripos($p, $n) === 0) { $ct++; } } } //phew -- if name is (somewhat) compatible, assume we've found it if ($ct > 0) { $r['proxyUrl'] = $url; $r['proxyYear'] = $proxy_year; $r['nameLast'] = trim(LsString::spacesToSpace($name['nameLast'])); $r['proxyName'] = trim(LsString::spacesToSpace($name['nameFull'])); } } } } } } } } unset($r); } else { //Error response (eg. 404, 500, etc) $log = fopen($this->logFile, 'a'); fwrite($log, "Couldn't get " . $url . "\n"); fclose($log); } return $roster; }