protected function importGovernor($row)
{
$url = $this->_baseUrl . $row['url'];
if (!$this->browser->get($url)->responseIsError()) {
$text = $this->browser->getResponseText();
$text = LsHtml::replaceEntities($text);
//preg_match('/>Family\:<\/b>([^<]*)<br/is',$text,$family_arr);
$name = trim(str_ireplace('Gov.', '', $row['name']));
$this->printDebug('');
$this->printDebug($name . ':');
$governor = PersonTable::parseFlatName($name);
$governor->addExtension('PoliticalCandidate');
$governor->addExtension('ElectedRepresentative');
$governor->is_state = 1;
$similar = $governor->getSimilarEntitiesQuery(true)->execute();
foreach ($similar as $s) {
$sim_re = LsString::escapeStringForRegex($s->name_first);
$search_re = LsString::escapeStringForRegex($governor->name_first);
if (preg_match('/^' . $sim_re . '/su', $governor->name_first) == 0 && preg_match('/^' . $search_re . '/su', $s->name_first) == 0) {
continue;
}
$bio = $s->getExtendedBio();
if (preg_match('/\\bgovernor(ship)?\\b/isu', $bio)) {
$governor = $s;
$this->printDebug(' Found existing governor: ' . $s->name . ' ' . $s->id);
break;
}
}
$governor->save();
$this->printDebug($governor->id);
if (!$governor->start_date && preg_match('/>Born\\:<\\/b>([^<]*)<br/is', $text, $birth_arr)) {
$this->printDebug(' Birthdate: ' . $birth_arr[1]);
$governor->start_date = trim($birth_arr[1]);
}
if (!$governor->birthplace && preg_match('/>Birth State\\:<\\/b>([^<]*)<br/is', $text, $birth_state_arr)) {
$this->printDebug(' Birthplace: ' . trim($birth_state_arr[1]));
$governor->birthplace = trim($birth_state_arr[1]);
}
//PARTY MEMBERSHIP
if (preg_match('/>Party\\:<\\/b>([^<]*)<br/is', $text, $party_arr)) {
$party_str = $party_arr[1];
$this->printDebug(' Party: ' . $party_str);
if (stristr($party_str, 'Democrat')) {
$party = EntityTable::getByExtensionQuery('PoliticalParty')->addWhere('name = ?', 'Democratic Party')->fetchOne();
}
if (stristr($party_str, 'Republican')) {
$party = EntityTable::getByExtensionQuery('PoliticalParty')->addWhere('name = ?', 'Republican Party')->fetchOne();
}
if (isset($party) && $party && !$governor->party_id) {
$governor->Party = $party;
$governor->is_independent = false;
$this->printDebug(' Added membership in ' . $party);
} else {
if (stristr($party_str, 'Independent')) {
$governor->is_independent = true;
}
}
}
if (!$governor->summary && preg_match_all('/>([^<]{240,})/isu', $text, $bio_match)) {
$str = '';
foreach ($bio_match[1] as $b) {
if (!stristr($b, 'Javascript')) {
$str .= "\n\n" . $b;
}
}
$str = trim($str);
if (strlen($str)) {
$governor->summary = $str;
}
}
$governor->save();
$governor->addReference($url, null, $governor->getAllModifiedFields(), 'Governors Association');
//SCHOOLS
if (preg_match('/>School\\(s\\)\\:<\\/b>([^<]*)<br/is', $text, $school_arr)) {
$school_names = explode(';', trim($school_arr[1]));
if (count($school_names) == 1) {
$school_names = explode(',', $school_names[0]);
}
foreach ($school_names as $school_name) {
$school_name = trim($school_name);
if (!($school = EntityTable::getByExtensionQuery('School')->leftJoin('e.Alias a')->addWhere('e.name = ? or a.name = ?', array($school_name, $school_name))->fetchOne())) {
$school = new Entity();
$school->addExtension('Org');
$school->addExtension('School');
$school->name = $school_name;
$school->save();
$this->printDebug(' Added School: ' . $school_name);
}
$q = RelationshipTable::getByCategoryQuery('Education')->addWhere('entity1_id = ? and entity2_id = ?', array($governor->id, $school->id))->fetchOne();
if (!$q) {
$relationship = new Relationship();
$relationship->setCategory('Education');
$relationship->Entity1 = $governor;
$relationship->Entity2 = $school;
$relationship->is_current = 0;
$relationship->save();
$relationship->addReference($url, null, $relationship->getAllModifiedFields(), 'Governors Association');
$this->printDebug(' Added education: ' . $relationship->name);
}
}
}
//GOVERNOR OFFICE AND POSITION
$office_name = 'Office of the Governor of ' . $row['state'];
if (!($office = EntityTable::getByExtensionQuery('GovernmentBody')->addWhere('name = ?', $office_name)->fetchOne())) {
$office = new Entity();
$office->name = $office_name;
$office->addExtension('Org');
$office->addExtension('GovernmentBody');
$state = Doctrine::getTable('AddressState')->findOneByName($row['state']);
if ($state) {
$office->state_id = $state->id;
}
$office->save();
$office->addReference($url, null, $office->getAllModifiedFields(), 'Governors Association');
$this->printDebug(' Added office: ' . $office->name);
}
$q = RelationshipTable::getByCategoryQuery('Position')->addWhere('entity1_id = ? and entity2_id = ? and description1 = ?', array($governor->id, $office->id, 'Governor'))->fetchOne();
if (!$q) {
sort($row['years']);
$i = 0;
while ($i < count($row['years'])) {
$governorship = new Relationship();
$governorship->setCategory('Position');
$governorship->Entity1 = $governor;
$governorship->Entity2 = $office;
$governorship->description1 = 'Governor';
$governorship->start_date = $row['years'][$i];
$i++;
if (isset($row['years'][$i])) {
$governorship->end_date = $row['years'][$i];
$governorship->is_current = 0;
if (!$governor->blurb && !isset($row['years'][$i + 1])) {
$governor->blurb = 'Former Governor of ' . $row['state'];
}
} else {
$governorship->is_current = 1;
if (!$governor->blurb) {
$governor->blurb = 'Governor of ' . $row['state'];
}
}
$governor->save();
$i++;
$governorship->save();
$governorship->addReference($url, null, $governorship->getAllModifiedFields(), 'Governors Association');
$this->printDebug(' Added governorship: ' . $governorship->name);
}
}
//SPOUSE
if (preg_match('/>Spouse\\:<\\/b>(.*?)<br/is', $text, $spouse_arr)) {
$spouse = trim(LsHtml::stripTags($spouse_arr[1]));
$q = RelationshipTable::getByCategoryQuery('Family')->addWhere('entity1_id = ? or entity2_id = ?', array($governor->id, $governor->id))->fetchOne();
if (!$q && strlen($spouse)) {
$spouse = PersonTable::parseFlatName($spouse);
$spouse->save();
$this->printDebug(' Added spouse: ' . $spouse->name);
$relationship = new Relationship();
$relationship->setCategory('Family');
$relationship->Entity1 = $spouse;
$relationship->Entity2 = $governor;
$relationship->description1 = 'Spouse';
$relationship->description2 = 'Spouse';
$relationship->save();
$relationship->addReference($url, null, $relationship->getAllModifiedFields(), 'Governors Association');
$this->printDebug(' Added spouse relationship: ' . $relationship->name);
}
}
//ADDRESS --not working, malformed addresses
/*
if (preg_match('/>Address\:\s*<\/b>(.*?)<b>/is',$text,$address_arr))
{
$address = trim(str_replace('<br/>',', ',$address_arr[1]));
$this->printDebug($address);
if ($governor->Address->count() == 0 && $a = $governor->addAddress($address))
{
$this->printDebug(' Address: ' . $a);
$governor->save();
}
}*/
//PHONE NUMBER
if (preg_match('/>Phone\\(s\\)\\:<\\/b>([^<]*)<br/is', $text, $phone_arr)) {
$phone_number = trim($phone_arr[1]);
if (!$governor->Phone->count()) {
$phone = $governor->addPhone($phone_number);
$this->printDebug(' Phone: ' . $phone);
}
}
if (!$governor->Image->count() && preg_match('/<img .*?class\\="display" src\\="([^"]*)"/is', $text, $img_arr)) {
$url = $img_arr[1];
try {
$fileName = ImageTable::createFiles($url, $governor->name_first);
} catch (Exception $e) {
$fileName = null;
}
if ($fileName) {
//insert image record
$image = new Image();
$image->filename = $fileName;
$image->entity_id = $governor->id;
$image->title = $governor->name;
$image->caption = 'From Governors Association website';
$image->is_featured = true;
$image->is_free = false;
$image->url = $url;
$image->save();
$this->printDebug("Imported image: " . $image->filename);
}
}
}
}
private function findPersonBio($page, $person, $org)
{
//$this->printDebug('');
$name_re = LsString::escapeStringForRegex($person->name_last);
if (preg_match('/<title>([^<]*)<\\/title>/is', $page, $match)) {
if (stristr($match[1], $person->name_last) && stristr($match[1], $person->name_first) && strlen($person->name_first) > 2) {
$name_re .= '|' . LsString::escapeStringForRegex($person->name_first);
}
}
$layout_tags = implode('|', LsHtml::$layoutTags);
$re2 = '/>([^<]*?(' . $name_re . ')(\\s|,|<)(.*?))<\\/?(' . $layout_tags . ')/is';
$re = $re2 . 'u';
//$this->printDebug($re);
$bio_match = null;
if (preg_match_all($re, $page, $matches) || preg_match_all($re2, $page, $matches)) {
//$this->printDebug('matches found');
$arr = array();
$most_reqs = 0;
$qual = false;
$news = false;
foreach ($matches[1] as $match) {
if (stristr($match, '}') || stristr($match, '{') || preg_match('/\\svar\\s/is', $match)) {
//$this->printDebug('FAILED - curly brackets');
continue;
}
$str = LsHtml::replaceEntities($match);
$str = LsHtml::stripTags($str, '');
$str = trim(LsString::spacesToSpace($str));
$this->printDebug(strlen($str));
if (strlen($str) > 3000) {
$this->printDebug('FAILED - str too long');
continue;
}
if (preg_match('/(^|\\b)(' . $name_re . ')\\b/is', $str) == 0) {
$this->printDebug($match . 'FAILED - no name match');
continue;
}
$word_count = count(explode(' ', $str));
if ($word_count < 12) {
$this->printDebug('FAILED - str not long enough');
continue;
} else {
if (stristr($str, 'announce') || stristr($str, 'today') || stristr($str, '—') || stristr($str, '–') || preg_match('/^[^\\-]{0,100}\\-(\\-|\\s)/is', $str)) {
$news = true;
$this->printDebug('FAILED: dash / announced / today');
} else {
if (preg_match('/(^|\\s)([\'"”])([^\\1]+)\\1/is', $str, $qm) && count(explode(' ', $qm[0])) > 6) {
$news = true;
$this->printDebug('FAILED: quote');
} else {
if (preg_match_all('/\\s(\\p{Ll})+\\b/su', $str, $lcm) < 5) {
$this->printDebug('FAILED: not enough lowercase');
} else {
$bio_words = PersonTable::$commonBioWords;
if (in_array('Lobbyist', $person->getExtensions())) {
$bio_words = array_merge($bio_words, LobbyistTable::$commonBioWords);
}
$bio_words = implode('|', $bio_words);
$bio_word_ct = preg_match_all('/\\s(' . $bio_words . ')\\s/is', $str, $matches);
$str = trim($str);
if (preg_match('/\\.$/is', $str) == 0) {
$this->printDebug('no period at end of string');
} else {
if ($bio_word_ct > 1) {
$news = false;
$qual = true;
$arr[] = $str;
} else {
$this->printDebug('less than 2 bio words');
if ($news == false) {
$str = preg_replace('/^[\\,\\.\\:\\;]\\s*/su', '', $str);
$arr[] = $str;
//array('str' => $str, 'bio_words' => $bio_word_ct);
}
}
}
}
}
}
//$this->printDebug('');
}
}
if ($qual) {
$arr = array_unique($arr);
$ret = false;
$bio = implode("\n\n", $arr);
//$this->printDebug($name_re);
if (strlen($bio) < 3000 && LsString::withinN($bio, '(' . $name_re . ')', '(is|was|holds|led|has|had|provides|practices|served|leads)', 2)) {
if (preg_match('/^.*?\\b(' . $name_re . ')\\b/is', $bio, $m) && count(explode(' ', $m[0])) < 20) {
$ret = true;
$this->printDebug('SUCCESS');
}
} else {
$this->printDebug('within N failed !!!!');
}
$org_test = true;
if ($ret && stristr($org->name, $person->name_last)) {
$org_test = false;
if (strlen($person->name_first) > 1) {
if (preg_match('/([^\\s]+\\s+){0,14}/is', $arr[0], $beg_match)) {
$nf_re = LsString::escapeStringForRegex($person->name_first);
if (preg_match('/\\b' . $nf_re . '\\b/is', $beg_match[0]) || preg_match('/\\b(Mr|Mrs|Ms)\\b/su', $arr[0])) {
$org_test = true;
//$this->printDebug('PASSED FIRST NAME TEST');
}
}
} else {
if (preg_match('/\\b(he|she|him|her|his|mr|ms|mrs)\\b/is', $arr[0])) {
$org_test = true;
//$this->printDebug('PASSED POSSESSIVE TEST');
}
}
}
if ($ret && $org_test) {
return $bio;
}
}
} else {
$this->printDebug('no matches found');
}
return false;
}
public function __construct($row)
{
$this->id = $row[1];
$this->name = trim(LsHtml::replaceEntities($row[2]));
$this->lifespan = trim(LsHtml::replaceEntities($row[3]));
$this->type = trim($row[4]);
$this->party = trim($row[5]);
$this->state = trim($row[6]);
list($this->termStart, $this->termEnd) = explode('-', trim($row[7]));
}
public function getSummary($str, Entity $e)
{
$str = LsHtml::replaceEntities($str);
$name_re = array();
$name_re[] = $e->getNameRegex();
if ($e->name_nick && $e->name_nick != '') {
$name_re[] = LsString::escapeStringForRegex($e->name_nick);
}
$name_re = implode('|', $name_re);
$style_tags = implode('|', LsHtml::$fontStyleTags);
$layout_tags = implode('|', LsHtml::$layoutTags);
$re = '/((' . $name_re . ')(.*?))<\\/?(' . $layout_tags . ')/isu';
$this->printDebug($re);
$results = null;
if (preg_match_all($re, $str, $matches)) {
$results = $matches[1];
foreach ($results as $result) {
$result = LsString::spacesToSpace(LsHtml::stripTags($result));
$this->printDebug($result);
}
}
return $results;
}
private function getProxy()
{
$text = null;
$this->printDebug($this->corp->name);
$url = "http://searchwww.sec.gov/EDGARFSClient/jsp/EDGAR_Query_Result.jsp?startDoc=1&queryString=&queryForm=DEF+14A&isAdv=1&queryCik=" . $this->corp->sec_cik . "&numResults=10#topAnchor";
$years = implode('|', $this->years);
if ($this->browser->get($url)->responseIsError()) {
echo "Couldn't get " . $url . "\n";
return;
}
$re = '/(' . $years . ')<\\/i>(<[^>]*>){2}<a[^\']+\'([^\']+)(?<=\\.htm)\'[^>]*>([^<]*)</isu';
$text = $this->browser->getResponseText();
//echo $text;
$matched = preg_match_all($re, $text, $matches, PREG_SET_ORDER);
if ($matched > 0) {
foreach ($matches as $match) {
if (stristr($match[3], 'def14') !== false || stristr($match[4], 'def 14') !== false) {
$this->year = $match[1];
//$this->printDebug($this->year);
$this->url = $match[3];
break;
}
}
if ($this->browser->get($this->url)->responseIsError()) {
echo "Couldn't get " . $this->url . "\n";
return;
}
$this->printDebug($this->url);
$text = $this->browser->getResponseText();
$text = LsHtml::replaceEntities($text);
$text = LsString::utf8TransUnaccent($text);
$this->text = $text;
}
}
public function parseResults($match)
{
if (isset($match['bio'])) {
$bio_dirty = LsHtml::replaceEntities(LsString::spacesToSpace(LsHtml::stripTags($match['bio'], "; ")));
$bio_dirty = preg_replace('/(\\;\\s)+/is', '; ', $bio_dirty);
}
foreach ($match as $k => &$m) {
$m = LsHtml::replaceEntities(LsString::spacesToSpace(LsHtml::stripTags($m, " ")));
}
if (isset($match['name'])) {
$name = $match['name'];
$bio = '';
if (isset($match['bio'])) {
$bio = $match['bio'];
}
} else {
return;
}
$this->printDebug("_________________________\n\nname: " . $name . "\n");
$this->printDebug("bio: " . $bio . "\n");
$accept = strtolower($this->readline('Process this entity? (n to skip) '));
if ($accept == 'n' || $accept == 'no') {
return false;
}
if (!$this->org_org) {
if ($this->last_first) {
$entity = PersonTable::parseCommaName($name);
} else {
$entity = PersonTable::parseFlatName($name);
}
$similar_entities = PersonTable::getSimilarQuery2($entity)->execute();
} else {
$entity = new Entity();
$entity->addExtension('Org');
foreach ($this->org_extensions as $ext) {
$entity->addExtension($ext);
}
$entity->setEntityField('name', $name);
$name = trim($name);
$name = str_replace('.', '', $name);
$similar_entities = OrgTable::getSimilarQuery($entity)->execute();
}
$matched = false;
foreach ($similar_entities as $similar_entity) {
if ($similar_entity['primary_ext'] == 'Person') {
$this->printDebug(' POSSIBLE MATCH: ' . $similar_entity->name . ' (Orgs :: ' . $similar_entity->getRelatedOrgsSummary() . " Bio :: {$similar_entity->summary})");
} else {
$this->printDebug(' POSSIBLE MATCH: ' . $similar_entity->name . ' (Summary :: ' . $similar_entity->summary . ')');
}
$accept = $this->readline(' Is this the same entity? (y or n)');
$attempts = 1;
while ($accept != 'y' && $accept != 'n' && $attempts < 5) {
$accept = $this->readline(' Is this the same entity? (y or n) ');
$attempts++;
}
if ($accept == 'y') {
$entity = $similar_entity;
$matched = true;
$this->printDebug(' [accepted]');
//sleep(1);
break;
} else {
if ($accept == 'break') {
break;
}
}
}
$created = false;
if (!$matched) {
if ($entity->getPrimaryExtension() == 'Person') {
$this->printDebug(' New person: ' . $entity->name_first . ' ' . $entity->name_last);
} else {
$this->printDebug(' New org: ' . $entity->name);
}
$accept = $this->readline(' create this new entity? (y or n) ');
$attempts = 1;
while ($accept != 'y' && $accept != 'n' && $attempts < 5) {
$accept = $this->readline(' create this new entity? (y or n) ');
$attempts++;
}
if ($accept == 'y') {
if ($entity->getPrimaryExtension() == 'Person') {
$this->printDebug("\n Bio: {$bio} \n");
$accept = $this->readline(' Add this bio? (y or n) ');
$attempts = 1;
while ($accept != 'y' && $accept != 'n' && $attempts < 5) {
$accept = $this->readline(' add this bio? (y or n) ');
$attempts++;
}
if ($accept == 'y') {
$entity->summary = $bio;
}
}
$entity->save();
$entity->addReference($this->url, null, null, $this->url_name);
$created = true;
$this->printDebug(' ' . $entity->name . ' saved');
//sleep(1);
}
}
if (($matched || $created) && $entity->getPrimaryExtension() == 'Person') {
$accept = $this->readline("Parse above bio for possible relationships? (y or n) ");
$attempts = 1;
while ($accept != 'y' && $accept != 'n' && $attempts < 5) {
$accept = $this->readline("Parse above bio for possible relationships? (y or n) ");
$attempts++;
}
if ($accept == 'y') {
$names = $entity->parseBio($bio_dirty);
$this->printDebug(" Orgs that {$entity} has a position at?");
foreach ($names as $name) {
$exists = false;
$name = trim($name);
$accept = $this->readline(" > {$name} :: an org? (y or n or b to break) ");
$attempts = 1;
$accept = strtolower($accept);
while ($accept != 'y' && $accept != 'n' && $accept != 'b' && $attempts < 5) {
$accept = $this->readline(" {$name} :: an org? (y or n or b to break) ");
$accept = strtolower($accept);
$attempts++;
}
if ($accept == 'b') {
break;
} else {
if ($accept == 'y') {
$this->printDebug(' .....looking for names.....');
$orgs = EntityTable::getByExtensionAndNameQuery('Org', $name)->limit(10)->execute();
$related_org = null;
foreach ($orgs as $org) {
$q = LsDoctrineQuery::create()->from('Relationship r')->where('entity1_id = ? and entity2_id = ?', array($entity->id, $org->id))->fetchOne();
if ($q) {
$this->printDebug(' Position already exists, skipping...');
$exists = true;
break;
}
$accept = $this->readline(" Create a position relationship between {$entity->name} and {$org->name}? (y or n) ");
$attempts = 1;
while ($accept != 'y' && $accept != 'n' && $attempts < 5) {
$accept = $this->readline(" Create a position relationship between {$entity->name} and {$org->name}? (y or n) ");
$attempts++;
}
if ($accept == 'y') {
$related_org = $org;
break;
}
}
if (!$related_org && !$exists) {
$accept = $this->readline(" couldn't find org, should this one be created: {$name} (y or n) ");
while ($accept != 'y' && $accept != 'n' && $attempts < 5) {
$accept = $this->readline(" couldn't find org, should this one be created: {$name} (y or n) ");
$attempts++;
}
if ($accept == 'y') {
$related_org = new Entity();
$related_org->addExtension('Org');
$related_org->name = preg_replace('/\\.(?!com)/i', '', $name);
$extensions = $this->readline(" what extensions should this org get? (eg 'Business, LobbyingFirm, LawFirm') ");
$extensions = preg_split('/\\,\\s*/isu', $extensions, -1, PREG_SPLIT_NO_EMPTY);
try {
foreach ($extensions as $extension) {
$related_org->addExtension($extension);
}
$related_org->save();
$related_org->addReference($this->url, null, null, $this->url_name);
} catch (Exception $e) {
$this->printDebug(' !!! problems with org creation, skipping');
$related_org = null;
}
}
}
if ($related_org) {
$q = LsDoctrineQuery::create()->from('Relationship r')->where('r.entity1_id = ? and r.entity2_id = ? and r.category_id = ?', array($entity->id, $related_org->id, 1))->fetchOne();
if ($q) {
$this->printDebug(' (relationship already found, skipping...)');
continue;
}
$relationship = new Relationship();
$relationship->Entity1 = $entity;
$relationship->Entity2 = $related_org;
$relationship->setCategory('Position');
$title = $this->readline(" Title for this position relationship? (<enter> to skip) ");
if (strlen($title) > 2) {
$relationship->description1 = $title;
}
$current = strtolower($this->readline(" Is the relationship current? (y or n or <enter> to skip) "));
if (in_array($current, array('y', 'yes'))) {
$relationship->is_current = 1;
} else {
if (in_array($current, array('n', 'no'))) {
$relationship->is_current = 0;
}
}
$board = strtolower($this->readline(" Is the relationship a board position? (y or n or <enter> to skip) "));
if (in_array($board, array('y', 'yes'))) {
$relationship->is_board = 1;
} else {
if (in_array($board, array('n', 'no'))) {
$relationship->is_board = 0;
}
}
$relationship->save();
$relationship->addReference($this->url, null, null, $this->url_name);
$this->printDebug(" Relationship saved: {$relationship}");
}
}
}
}
}
}
if ($matched || $created) {
if ($this->list) {
$q = LsDoctrineQuery::create()->from('LsListEntity l')->where('l.entity_id = ? and l.list_id = ?', array($entity->id, $this->list->id))->fetchOne();
if (!$q) {
$le = new LsListEntity();
$le->Entity = $entity;
$le->LsList = $this->list;
if (isset($match['rank'])) {
if (preg_match('/(\\d+)/isu', $match['rank'], $m)) {
$le->rank = $m[1];
}
}
$le->save();
$this->printDebug('List membership saved');
}
}
if ($this->org) {
$q = LsDoctrineQuery::create()->from('Relationship r')->where('r.entity1_id = ? and r.entity2_id = ? and r.category_id = ?', array($entity->id, $this->org->id, 1))->fetchOne();
if ($q) {
$this->printDebug(' (relationship already found, skipping...)');
return;
}
$relationship = new Relationship();
$relationship->Entity1 = $entity;
$relationship->Entity2 = $this->org;
$relationship->setCategory($this->relationship_category);
if ($this->description1) {
$relationship->description1 = $this->description1;
} else {
$description = $this->readline(" what description to give this relationship ({$relationship}) ? (less than 3 chars will skip)");
if (strlen($description) > 2) {
$relationship->description1 = $description;
}
}
if ($this->relationship_category == 'Position') {
$relationship->is_board = $this->is_board;
} else {
if ($this->relationship_category == 'Donation') {
if ($this->amount) {
$relationship->amount = $this->amount;
} else {
$amount = $this->readline(" what amount ({$relationship}) ? (less than 3 chars will skip)");
if (strlen($amount) > 1) {
$relationship->amount = $amount;
}
}
}
}
$relationship->save();
$relationship->addReference($this->url, null, null, $this->url_name);
$this->printDebug(" Relationship saved: {$relationship}");
}
}
//dump history
if (isset($match['affiliation1'])) {
$affiliation = $match['affiliation'];
//$this->printDebug($affiliation);
}
}
function __construct($text)
{
$text = LsHtml::replaceEntities($text);
$text = LsString::utf8TransUnaccent($text);
$this->text = $text;
}
static function getHtmlPersonNames($text)
{
$name_matches = array();
$re = '/>\\s*\\p{Lu}\'?(\\p{L}+|\\.)?\\s+\\p{Lu}\\.?\\s+\\p{Lu}\\p{L}+(\\,?\\s+\\p{Lu}\\p{L}{1,4}\\.?)?/su';
$re2 = '/>\\s*(\\p{Lu}\'?(\\p{L}+|\\.)?\\s+(\\p{Lu}\'?(\\s+|\\p{L}+\\s+|\\.\\s*)?){0,2}\\p{Lu}\'?\\p{L}+(\\-\\p{Lu}\'?\\p{L}+)?(\\,?\\s+\\p{Lu}\\p{L}{1,4}\\.?)?)\\**\\s*</su';
$re3 = '/>\\s*(\\p{Lu}\'?\\p{L}+(\\-\\p{Lu}\'?\\p{L}+)?\\,\\s+(\\p{Lu}\'?(\\p{L}+|\\.)?(\\s+(\\p{Lu}\'?(\\s+|\\p{L}+\\s+|\\.\\s*)?){0,2})?)(\\,?\\s+\\p{Lu}\\p{L}{1,4}\\.?)?)\\**\\s*</su';
$text = LsHtml::replaceEntities($text);
$name_matches = array();
if (preg_match_all($re2, $text, $matches, PREG_OFFSET_CAPTURE)) {
//LOOP THROUGH MATCHES TO CONFIRM NAMES
for ($i = 0; $i < count($matches[1]); $i++) {
$m = $matches[1][$i];
//echo $m[0] . "\n";
$is_name = false;
if (preg_match('/\\s+\\p{Lu}\\.?\\s/', $m[0])) {
//echo ' * initial' . "\n";
$is_name = true;
}
$parts = LsString::split(trim($m[0]));
//ADD NAME TO MATCH LIST IF IT FITS CONDITIONS
if (in_array($parts[0], LsLanguage::$commonFirstNames)) {
//echo ' * first name' . "\n";
$is_name = true;
}
$q = LsDoctrineQuery::create()->from('Person p')->where('p.name_first = ?', $parts[0]);
if ($q->count() > 0) {
//echo ' LS name' . "\n";
$is_name = true;
}
if ($is_name) {
$name_matches[] = $m[0];
}
/*
if ($i != 0)
{
$beg = $matches[1][$i-1][1];
$tweenstr = substr($text,$beg, $m[1] - $beg);
//echo ' tag count: ' . LsHtml::tagCount($tweenstr) . "\n";
}
preg_match('/^[^\s]+\s/su',trim($m[0]),$match);
$tags = LsHtml::getSurroundingTags($text,$m[1],3);*/
}
}
if (preg_match_all($re3, $text, $matches, PREG_OFFSET_CAPTURE)) {
for ($i = 0; $i < count($matches[1]); $i++) {
$m = $matches[1][$i];
//echo $m[0] . "\n";
$person = PersonTable::parseCommaName($m[0]);
$name_matches[] = $person->getFullName(false);
}
}
return $name_matches;
}
public function checkBoardPage($board_rels)
{
$goog = new LsGoogle();
$goog->setQuery($this->entity->name . " board");
$results = $goog->execute();
$results = $goog->parseSearchResults($results);
if (count($results)) {
$url = $results[0]['unescapedUrl'];
if (stripos($url, "yahoo.com")) {
$url = $results[1]['unescapedUrl'];
}
$this->printDebug($url);
try {
if (!$this->browser->get($url)->responseIsError()) {
$text = $this->browser->getResponseText();
$text = LsHtml::replaceEntities($text);
$status_arr = array();
$unique_arr = array();
foreach ($board_rels as $br) {
$found = 0;
//$this->printDebug($br->Entity1->getNameRegex());
$regexes = $br->Entity1->getNameRegexes();
foreach ($regexes as $regex) {
if (preg_match_all($regex, $text, $matches, PREG_SET_ORDER | PREG_OFFSET_CAPTURE)) {
//var_dump($matches);
$found = 1;
if (!in_array($br->entity1_id, $unique_arr)) {
$unique_arr[] = $br->entity1_id;
}
break;
}
}
$status_arr[] = $found;
$this->printDebug($br->Entity1->name . " > " . $found);
}
if (count($unique_arr) > 1) {
$this->printDebug("\tenough board member names found to mark as current or not");
for ($i = 0; $i < count($status_arr); $i++) {
$br = $board_rels[$i];
if ($status_arr[$i] == 1) {
$br->is_current = 1;
} else {
$br->is_current = 0;
}
if (!$this->testMode) {
$br->save();
$br->addReference($url, null, null, $this->entity->name . ' board', null, null);
}
}
return 1;
} else {
return 0;
}
}
} catch (Exception $e) {
return 0;
}
} else {
return -1;
}
}
public function getCleanFirstParagraph()
{
if ($this->_paragraphs) {
return null;
}
$first = $this->_paragraphs[0];
$first = LsString::spacesToSpace(LsHtml::replaceEntities(LsHtml::stripTags($first)));
return $first;
}
private function getProxyData($roster, $url, $proxy_year)
{
echo "fetching data from proxy at {$url} \n\n";
$people_count = 0;
if (!$this->browser->get($url)->responseIsError()) {
$this->proxyText = $this->browser->getResponseText();
$this->proxyText = LsHtml::replaceEntities($this->proxyText, ENT_QUOTES, 'UTF-8');
$this->proxyText = LsString::utf8TransUnaccent($this->proxyText);
foreach ($roster as &$r) {
//make sure this is not form 4 data for a corporation, continue to the next if it is
if ($r['officerTitle'] == '' && $r['isDirector'] != 1 && strtoupper($r['isDirector']) != strtoupper('true')) {
continue;
}
//echo $re;
$parts = preg_split("/[\\s|\\.]+/", $r['personName'], -1, PREG_SPLIT_NO_EMPTY);
//first word, but has to be part of last name because form4 names are in format RUBIN ROBERT E
$last = trim($parts[0]);
//sometimes O'LEARY can appear as O LEARY in the form 4
if (strlen($last) == 1) {
$r['personName'] = $last . substr($r['personName'], 2);
$parts = preg_split("/[\\s|\\.]+/", $r['personName'], -1, PREG_SPLIT_NO_EMPTY);
$last = trim($parts[0]);
}
//prepare regex to match occurrences of full name
//case insensitive to accommodate for various irregularities in names
$re = LsLanguage::buildLooseNameRegex($r['personName']);
$offset = 0;
$found = true;
//use stripos (much faster than regex) to find occurrences of the first word in the form 4 name (assumed to be part of the last name)
//needs to be case insensitive
//continue searching for last name in proxy until a matching full name (proxyName) is found
while (!isset($r['proxyName']) && $found !== false) {
$found = stripos($this->proxyText, $last, $offset);
//$this->printDebug('found at pos:' . $found);
$offset = $found + 1;
if ($found !== false) {
$str = substr($this->proxyText, $found - 70, 120);
//$this->printDebug('found string: ' . $str);
//$this->printDebug($re);
preg_match_all($re, $str, $matches, PREG_OFFSET_CAPTURE | PREG_SET_ORDER);
//$this->printDebug('matchcount is ' . count($matches));
foreach ($matches as $match) {
if (stristr($match[1][0], '=')) {
continue;
}
//since we may or may not be working with the full last name, use getLastName to return full last name
$new_last = $this->getLastName($r['personName'], $match[1][0]);
if ($new_last) {
//if last name produced by case insensitive search has no capital letters, not a match
if (preg_match('/\\p{Lu}/su', $new_last) == 0) {
continue;
}
//now that we have a last name, pull the full name from the string
$name = LsLanguage::getNameWithLast($match[0][0], $new_last);
if ($name) {
$parts = preg_split('/\\s+/isu', $name['nameStart'], -1, PREG_SPLIT_NO_EMPTY);
$non_prefixes = array_diff($parts, PersonTable::$nameParsePrefixes);
//if all we've found are matching prefixes, not a match
if (count($non_prefixes) == 0) {
continue;
} else {
$name1_parts = preg_split('/\\s+/', $r['personName'], -1, PREG_SPLIT_NO_EMPTY);
$ct = 0;
//compatibility check to correct for vagueness of regex
foreach ($non_prefixes as $n) {
foreach ($name1_parts as $p) {
if (stripos($n, $p) === 0 || stripos($p, $n) === 0) {
$ct++;
}
}
}
//phew -- if name is (somewhat) compatible, assume we've found it
if ($ct > 0) {
$r['proxyUrl'] = $url;
$r['proxyYear'] = $proxy_year;
$r['nameLast'] = trim(LsString::spacesToSpace($name['nameLast']));
$r['proxyName'] = trim(LsString::spacesToSpace($name['nameFull']));
}
}
}
}
}
}
}
}
unset($r);
} else {
//Error response (eg. 404, 500, etc)
$log = fopen($this->logFile, 'a');
fwrite($log, "Couldn't get " . $url . "\n");
fclose($log);
}
return $roster;
}
