/**
* get donor info
*/
private function generateDonor($text)
{
$text_arr = explode("<BR>", $text);
//var_dump($text_arr[0]);
$donor = $this->generatePerson(LsHtml::stripTags($text_arr[0], ''));
$address_arr = LsLanguage::parseCityStatePostal($text_arr[1]);
$a = new Address();
$a->street1 = isset($address_arr['street1']) ? $address_arr['street1'] : null;
$a->street2 = isset($address_arr['street2']) ? $address_arr['street2'] : null;
$a->city = $address_arr['city'];
if ($state = AddressStateTable::retrieveByText($address_arr['state'])) {
$a->State = $state;
}
$a->postal = $address_arr['zip'];
$donor->addAddress($a);
$donor->summary = strip_tags(trim($text_arr[2]));
return $donor;
}
private function findPersonBio($page, $person, $org)
{
//$this->printDebug('');
$name_re = LsString::escapeStringForRegex($person->name_last);
if (preg_match('/<title>([^<]*)<\\/title>/is', $page, $match)) {
if (stristr($match[1], $person->name_last) && stristr($match[1], $person->name_first) && strlen($person->name_first) > 2) {
$name_re .= '|' . LsString::escapeStringForRegex($person->name_first);
}
}
$layout_tags = implode('|', LsHtml::$layoutTags);
$re2 = '/>([^<]*?(' . $name_re . ')(\\s|,|<)(.*?))<\\/?(' . $layout_tags . ')/is';
$re = $re2 . 'u';
//$this->printDebug($re);
$bio_match = null;
if (preg_match_all($re, $page, $matches) || preg_match_all($re2, $page, $matches)) {
//$this->printDebug('matches found');
$arr = array();
$most_reqs = 0;
$qual = false;
$news = false;
foreach ($matches[1] as $match) {
if (stristr($match, '}') || stristr($match, '{') || preg_match('/\\svar\\s/is', $match)) {
//$this->printDebug('FAILED - curly brackets');
continue;
}
$str = LsHtml::replaceEntities($match);
$str = LsHtml::stripTags($str, '');
$str = trim(LsString::spacesToSpace($str));
$this->printDebug(strlen($str));
if (strlen($str) > 3000) {
$this->printDebug('FAILED - str too long');
continue;
}
if (preg_match('/(^|\\b)(' . $name_re . ')\\b/is', $str) == 0) {
$this->printDebug($match . 'FAILED - no name match');
continue;
}
$word_count = count(explode(' ', $str));
if ($word_count < 12) {
$this->printDebug('FAILED - str not long enough');
continue;
} else {
if (stristr($str, 'announce') || stristr($str, 'today') || stristr($str, '—') || stristr($str, '–') || preg_match('/^[^\\-]{0,100}\\-(\\-|\\s)/is', $str)) {
$news = true;
$this->printDebug('FAILED: dash / announced / today');
} else {
if (preg_match('/(^|\\s)([\'"”])([^\\1]+)\\1/is', $str, $qm) && count(explode(' ', $qm[0])) > 6) {
$news = true;
$this->printDebug('FAILED: quote');
} else {
if (preg_match_all('/\\s(\\p{Ll})+\\b/su', $str, $lcm) < 5) {
$this->printDebug('FAILED: not enough lowercase');
} else {
$bio_words = PersonTable::$commonBioWords;
if (in_array('Lobbyist', $person->getExtensions())) {
$bio_words = array_merge($bio_words, LobbyistTable::$commonBioWords);
}
$bio_words = implode('|', $bio_words);
$bio_word_ct = preg_match_all('/\\s(' . $bio_words . ')\\s/is', $str, $matches);
$str = trim($str);
if (preg_match('/\\.$/is', $str) == 0) {
$this->printDebug('no period at end of string');
} else {
if ($bio_word_ct > 1) {
$news = false;
$qual = true;
$arr[] = $str;
} else {
$this->printDebug('less than 2 bio words');
if ($news == false) {
$str = preg_replace('/^[\\,\\.\\:\\;]\\s*/su', '', $str);
$arr[] = $str;
//array('str' => $str, 'bio_words' => $bio_word_ct);
}
}
}
}
}
}
//$this->printDebug('');
}
}
if ($qual) {
$arr = array_unique($arr);
$ret = false;
$bio = implode("\n\n", $arr);
//$this->printDebug($name_re);
if (strlen($bio) < 3000 && LsString::withinN($bio, '(' . $name_re . ')', '(is|was|holds|led|has|had|provides|practices|served|leads)', 2)) {
if (preg_match('/^.*?\\b(' . $name_re . ')\\b/is', $bio, $m) && count(explode(' ', $m[0])) < 20) {
$ret = true;
$this->printDebug('SUCCESS');
}
} else {
$this->printDebug('within N failed !!!!');
}
$org_test = true;
if ($ret && stristr($org->name, $person->name_last)) {
$org_test = false;
if (strlen($person->name_first) > 1) {
if (preg_match('/([^\\s]+\\s+){0,14}/is', $arr[0], $beg_match)) {
$nf_re = LsString::escapeStringForRegex($person->name_first);
if (preg_match('/\\b' . $nf_re . '\\b/is', $beg_match[0]) || preg_match('/\\b(Mr|Mrs|Ms)\\b/su', $arr[0])) {
$org_test = true;
//$this->printDebug('PASSED FIRST NAME TEST');
}
}
} else {
if (preg_match('/\\b(he|she|him|her|his|mr|ms|mrs)\\b/is', $arr[0])) {
$org_test = true;
//$this->printDebug('PASSED POSSESSIVE TEST');
}
}
}
if ($ret && $org_test) {
return $bio;
}
}
} else {
$this->printDebug('no matches found');
}
return false;
}
protected function importGovernor($row)
{
$url = $this->_baseUrl . $row['url'];
if (!$this->browser->get($url)->responseIsError()) {
$text = $this->browser->getResponseText();
$text = LsHtml::replaceEntities($text);
//preg_match('/>Family\:<\/b>([^<]*)<br/is',$text,$family_arr);
$name = trim(str_ireplace('Gov.', '', $row['name']));
$this->printDebug('');
$this->printDebug($name . ':');
$governor = PersonTable::parseFlatName($name);
$governor->addExtension('PoliticalCandidate');
$governor->addExtension('ElectedRepresentative');
$governor->is_state = 1;
$similar = $governor->getSimilarEntitiesQuery(true)->execute();
foreach ($similar as $s) {
$sim_re = LsString::escapeStringForRegex($s->name_first);
$search_re = LsString::escapeStringForRegex($governor->name_first);
if (preg_match('/^' . $sim_re . '/su', $governor->name_first) == 0 && preg_match('/^' . $search_re . '/su', $s->name_first) == 0) {
continue;
}
$bio = $s->getExtendedBio();
if (preg_match('/\\bgovernor(ship)?\\b/isu', $bio)) {
$governor = $s;
$this->printDebug(' Found existing governor: ' . $s->name . ' ' . $s->id);
break;
}
}
$governor->save();
$this->printDebug($governor->id);
if (!$governor->start_date && preg_match('/>Born\\:<\\/b>([^<]*)<br/is', $text, $birth_arr)) {
$this->printDebug(' Birthdate: ' . $birth_arr[1]);
$governor->start_date = trim($birth_arr[1]);
}
if (!$governor->birthplace && preg_match('/>Birth State\\:<\\/b>([^<]*)<br/is', $text, $birth_state_arr)) {
$this->printDebug(' Birthplace: ' . trim($birth_state_arr[1]));
$governor->birthplace = trim($birth_state_arr[1]);
}
//PARTY MEMBERSHIP
if (preg_match('/>Party\\:<\\/b>([^<]*)<br/is', $text, $party_arr)) {
$party_str = $party_arr[1];
$this->printDebug(' Party: ' . $party_str);
if (stristr($party_str, 'Democrat')) {
$party = EntityTable::getByExtensionQuery('PoliticalParty')->addWhere('name = ?', 'Democratic Party')->fetchOne();
}
if (stristr($party_str, 'Republican')) {
$party = EntityTable::getByExtensionQuery('PoliticalParty')->addWhere('name = ?', 'Republican Party')->fetchOne();
}
if (isset($party) && $party && !$governor->party_id) {
$governor->Party = $party;
$governor->is_independent = false;
$this->printDebug(' Added membership in ' . $party);
} else {
if (stristr($party_str, 'Independent')) {
$governor->is_independent = true;
}
}
}
if (!$governor->summary && preg_match_all('/>([^<]{240,})/isu', $text, $bio_match)) {
$str = '';
foreach ($bio_match[1] as $b) {
if (!stristr($b, 'Javascript')) {
$str .= "\n\n" . $b;
}
}
$str = trim($str);
if (strlen($str)) {
$governor->summary = $str;
}
}
$governor->save();
$governor->addReference($url, null, $governor->getAllModifiedFields(), 'Governors Association');
//SCHOOLS
if (preg_match('/>School\\(s\\)\\:<\\/b>([^<]*)<br/is', $text, $school_arr)) {
$school_names = explode(';', trim($school_arr[1]));
if (count($school_names) == 1) {
$school_names = explode(',', $school_names[0]);
}
foreach ($school_names as $school_name) {
$school_name = trim($school_name);
if (!($school = EntityTable::getByExtensionQuery('School')->leftJoin('e.Alias a')->addWhere('e.name = ? or a.name = ?', array($school_name, $school_name))->fetchOne())) {
$school = new Entity();
$school->addExtension('Org');
$school->addExtension('School');
$school->name = $school_name;
$school->save();
$this->printDebug(' Added School: ' . $school_name);
}
$q = RelationshipTable::getByCategoryQuery('Education')->addWhere('entity1_id = ? and entity2_id = ?', array($governor->id, $school->id))->fetchOne();
if (!$q) {
$relationship = new Relationship();
$relationship->setCategory('Education');
$relationship->Entity1 = $governor;
$relationship->Entity2 = $school;
$relationship->is_current = 0;
$relationship->save();
$relationship->addReference($url, null, $relationship->getAllModifiedFields(), 'Governors Association');
$this->printDebug(' Added education: ' . $relationship->name);
}
}
}
//GOVERNOR OFFICE AND POSITION
$office_name = 'Office of the Governor of ' . $row['state'];
if (!($office = EntityTable::getByExtensionQuery('GovernmentBody')->addWhere('name = ?', $office_name)->fetchOne())) {
$office = new Entity();
$office->name = $office_name;
$office->addExtension('Org');
$office->addExtension('GovernmentBody');
$state = Doctrine::getTable('AddressState')->findOneByName($row['state']);
if ($state) {
$office->state_id = $state->id;
}
$office->save();
$office->addReference($url, null, $office->getAllModifiedFields(), 'Governors Association');
$this->printDebug(' Added office: ' . $office->name);
}
$q = RelationshipTable::getByCategoryQuery('Position')->addWhere('entity1_id = ? and entity2_id = ? and description1 = ?', array($governor->id, $office->id, 'Governor'))->fetchOne();
if (!$q) {
sort($row['years']);
$i = 0;
while ($i < count($row['years'])) {
$governorship = new Relationship();
$governorship->setCategory('Position');
$governorship->Entity1 = $governor;
$governorship->Entity2 = $office;
$governorship->description1 = 'Governor';
$governorship->start_date = $row['years'][$i];
$i++;
if (isset($row['years'][$i])) {
$governorship->end_date = $row['years'][$i];
$governorship->is_current = 0;
if (!$governor->blurb && !isset($row['years'][$i + 1])) {
$governor->blurb = 'Former Governor of ' . $row['state'];
}
} else {
$governorship->is_current = 1;
if (!$governor->blurb) {
$governor->blurb = 'Governor of ' . $row['state'];
}
}
$governor->save();
$i++;
$governorship->save();
$governorship->addReference($url, null, $governorship->getAllModifiedFields(), 'Governors Association');
$this->printDebug(' Added governorship: ' . $governorship->name);
}
}
//SPOUSE
if (preg_match('/>Spouse\\:<\\/b>(.*?)<br/is', $text, $spouse_arr)) {
$spouse = trim(LsHtml::stripTags($spouse_arr[1]));
$q = RelationshipTable::getByCategoryQuery('Family')->addWhere('entity1_id = ? or entity2_id = ?', array($governor->id, $governor->id))->fetchOne();
if (!$q && strlen($spouse)) {
$spouse = PersonTable::parseFlatName($spouse);
$spouse->save();
$this->printDebug(' Added spouse: ' . $spouse->name);
$relationship = new Relationship();
$relationship->setCategory('Family');
$relationship->Entity1 = $spouse;
$relationship->Entity2 = $governor;
$relationship->description1 = 'Spouse';
$relationship->description2 = 'Spouse';
$relationship->save();
$relationship->addReference($url, null, $relationship->getAllModifiedFields(), 'Governors Association');
$this->printDebug(' Added spouse relationship: ' . $relationship->name);
}
}
//ADDRESS --not working, malformed addresses
/*
if (preg_match('/>Address\:\s*<\/b>(.*?)<b>/is',$text,$address_arr))
{
$address = trim(str_replace('<br/>',', ',$address_arr[1]));
$this->printDebug($address);
if ($governor->Address->count() == 0 && $a = $governor->addAddress($address))
{
$this->printDebug(' Address: ' . $a);
$governor->save();
}
}*/
//PHONE NUMBER
if (preg_match('/>Phone\\(s\\)\\:<\\/b>([^<]*)<br/is', $text, $phone_arr)) {
$phone_number = trim($phone_arr[1]);
if (!$governor->Phone->count()) {
$phone = $governor->addPhone($phone_number);
$this->printDebug(' Phone: ' . $phone);
}
}
if (!$governor->Image->count() && preg_match('/<img .*?class\\="display" src\\="([^"]*)"/is', $text, $img_arr)) {
$url = $img_arr[1];
try {
$fileName = ImageTable::createFiles($url, $governor->name_first);
} catch (Exception $e) {
$fileName = null;
}
if ($fileName) {
//insert image record
$image = new Image();
$image->filename = $fileName;
$image->entity_id = $governor->id;
$image->title = $governor->name;
$image->caption = 'From Governors Association website';
$image->is_featured = true;
$image->is_free = false;
$image->url = $url;
$image->save();
$this->printDebug("Imported image: " . $image->filename);
}
}
}
}
public function getSummary($str, Entity $e)
{
$str = LsHtml::replaceEntities($str);
$name_re = array();
$name_re[] = $e->getNameRegex();
if ($e->name_nick && $e->name_nick != '') {
$name_re[] = LsString::escapeStringForRegex($e->name_nick);
}
$name_re = implode('|', $name_re);
$style_tags = implode('|', LsHtml::$fontStyleTags);
$layout_tags = implode('|', LsHtml::$layoutTags);
$re = '/((' . $name_re . ')(.*?))<\\/?(' . $layout_tags . ')/isu';
$this->printDebug($re);
$results = null;
if (preg_match_all($re, $str, $matches)) {
$results = $matches[1];
foreach ($results as $result) {
$result = LsString::spacesToSpace(LsHtml::stripTags($result));
$this->printDebug($result);
}
}
return $results;
}
public function parseResults($match)
{
if (isset($match['bio'])) {
$bio_dirty = LsHtml::replaceEntities(LsString::spacesToSpace(LsHtml::stripTags($match['bio'], "; ")));
$bio_dirty = preg_replace('/(\\;\\s)+/is', '; ', $bio_dirty);
}
foreach ($match as $k => &$m) {
$m = LsHtml::replaceEntities(LsString::spacesToSpace(LsHtml::stripTags($m, " ")));
}
if (isset($match['name'])) {
$name = $match['name'];
$bio = '';
if (isset($match['bio'])) {
$bio = $match['bio'];
}
} else {
return;
}
$this->printDebug("_________________________\n\nname: " . $name . "\n");
$this->printDebug("bio: " . $bio . "\n");
$accept = strtolower($this->readline('Process this entity? (n to skip) '));
if ($accept == 'n' || $accept == 'no') {
return false;
}
if (!$this->org_org) {
if ($this->last_first) {
$entity = PersonTable::parseCommaName($name);
} else {
$entity = PersonTable::parseFlatName($name);
}
$similar_entities = PersonTable::getSimilarQuery2($entity)->execute();
} else {
$entity = new Entity();
$entity->addExtension('Org');
foreach ($this->org_extensions as $ext) {
$entity->addExtension($ext);
}
$entity->setEntityField('name', $name);
$name = trim($name);
$name = str_replace('.', '', $name);
$similar_entities = OrgTable::getSimilarQuery($entity)->execute();
}
$matched = false;
foreach ($similar_entities as $similar_entity) {
if ($similar_entity['primary_ext'] == 'Person') {
$this->printDebug(' POSSIBLE MATCH: ' . $similar_entity->name . ' (Orgs :: ' . $similar_entity->getRelatedOrgsSummary() . " Bio :: {$similar_entity->summary})");
} else {
$this->printDebug(' POSSIBLE MATCH: ' . $similar_entity->name . ' (Summary :: ' . $similar_entity->summary . ')');
}
$accept = $this->readline(' Is this the same entity? (y or n)');
$attempts = 1;
while ($accept != 'y' && $accept != 'n' && $attempts < 5) {
$accept = $this->readline(' Is this the same entity? (y or n) ');
$attempts++;
}
if ($accept == 'y') {
$entity = $similar_entity;
$matched = true;
$this->printDebug(' [accepted]');
//sleep(1);
break;
} else {
if ($accept == 'break') {
break;
}
}
}
$created = false;
if (!$matched) {
if ($entity->getPrimaryExtension() == 'Person') {
$this->printDebug(' New person: ' . $entity->name_first . ' ' . $entity->name_last);
} else {
$this->printDebug(' New org: ' . $entity->name);
}
$accept = $this->readline(' create this new entity? (y or n) ');
$attempts = 1;
while ($accept != 'y' && $accept != 'n' && $attempts < 5) {
$accept = $this->readline(' create this new entity? (y or n) ');
$attempts++;
}
if ($accept == 'y') {
if ($entity->getPrimaryExtension() == 'Person') {
$this->printDebug("\n Bio: {$bio} \n");
$accept = $this->readline(' Add this bio? (y or n) ');
$attempts = 1;
while ($accept != 'y' && $accept != 'n' && $attempts < 5) {
$accept = $this->readline(' add this bio? (y or n) ');
$attempts++;
}
if ($accept == 'y') {
$entity->summary = $bio;
}
}
$entity->save();
$entity->addReference($this->url, null, null, $this->url_name);
$created = true;
$this->printDebug(' ' . $entity->name . ' saved');
//sleep(1);
}
}
if (($matched || $created) && $entity->getPrimaryExtension() == 'Person') {
$accept = $this->readline("Parse above bio for possible relationships? (y or n) ");
$attempts = 1;
while ($accept != 'y' && $accept != 'n' && $attempts < 5) {
$accept = $this->readline("Parse above bio for possible relationships? (y or n) ");
$attempts++;
}
if ($accept == 'y') {
$names = $entity->parseBio($bio_dirty);
$this->printDebug(" Orgs that {$entity} has a position at?");
foreach ($names as $name) {
$exists = false;
$name = trim($name);
$accept = $this->readline(" > {$name} :: an org? (y or n or b to break) ");
$attempts = 1;
$accept = strtolower($accept);
while ($accept != 'y' && $accept != 'n' && $accept != 'b' && $attempts < 5) {
$accept = $this->readline(" {$name} :: an org? (y or n or b to break) ");
$accept = strtolower($accept);
$attempts++;
}
if ($accept == 'b') {
break;
} else {
if ($accept == 'y') {
$this->printDebug(' .....looking for names.....');
$orgs = EntityTable::getByExtensionAndNameQuery('Org', $name)->limit(10)->execute();
$related_org = null;
foreach ($orgs as $org) {
$q = LsDoctrineQuery::create()->from('Relationship r')->where('entity1_id = ? and entity2_id = ?', array($entity->id, $org->id))->fetchOne();
if ($q) {
$this->printDebug(' Position already exists, skipping...');
$exists = true;
break;
}
$accept = $this->readline(" Create a position relationship between {$entity->name} and {$org->name}? (y or n) ");
$attempts = 1;
while ($accept != 'y' && $accept != 'n' && $attempts < 5) {
$accept = $this->readline(" Create a position relationship between {$entity->name} and {$org->name}? (y or n) ");
$attempts++;
}
if ($accept == 'y') {
$related_org = $org;
break;
}
}
if (!$related_org && !$exists) {
$accept = $this->readline(" couldn't find org, should this one be created: {$name} (y or n) ");
while ($accept != 'y' && $accept != 'n' && $attempts < 5) {
$accept = $this->readline(" couldn't find org, should this one be created: {$name} (y or n) ");
$attempts++;
}
if ($accept == 'y') {
$related_org = new Entity();
$related_org->addExtension('Org');
$related_org->name = preg_replace('/\\.(?!com)/i', '', $name);
$extensions = $this->readline(" what extensions should this org get? (eg 'Business, LobbyingFirm, LawFirm') ");
$extensions = preg_split('/\\,\\s*/isu', $extensions, -1, PREG_SPLIT_NO_EMPTY);
try {
foreach ($extensions as $extension) {
$related_org->addExtension($extension);
}
$related_org->save();
$related_org->addReference($this->url, null, null, $this->url_name);
} catch (Exception $e) {
$this->printDebug(' !!! problems with org creation, skipping');
$related_org = null;
}
}
}
if ($related_org) {
$q = LsDoctrineQuery::create()->from('Relationship r')->where('r.entity1_id = ? and r.entity2_id = ? and r.category_id = ?', array($entity->id, $related_org->id, 1))->fetchOne();
if ($q) {
$this->printDebug(' (relationship already found, skipping...)');
continue;
}
$relationship = new Relationship();
$relationship->Entity1 = $entity;
$relationship->Entity2 = $related_org;
$relationship->setCategory('Position');
$title = $this->readline(" Title for this position relationship? (<enter> to skip) ");
if (strlen($title) > 2) {
$relationship->description1 = $title;
}
$current = strtolower($this->readline(" Is the relationship current? (y or n or <enter> to skip) "));
if (in_array($current, array('y', 'yes'))) {
$relationship->is_current = 1;
} else {
if (in_array($current, array('n', 'no'))) {
$relationship->is_current = 0;
}
}
$board = strtolower($this->readline(" Is the relationship a board position? (y or n or <enter> to skip) "));
if (in_array($board, array('y', 'yes'))) {
$relationship->is_board = 1;
} else {
if (in_array($board, array('n', 'no'))) {
$relationship->is_board = 0;
}
}
$relationship->save();
$relationship->addReference($this->url, null, null, $this->url_name);
$this->printDebug(" Relationship saved: {$relationship}");
}
}
}
}
}
}
if ($matched || $created) {
if ($this->list) {
$q = LsDoctrineQuery::create()->from('LsListEntity l')->where('l.entity_id = ? and l.list_id = ?', array($entity->id, $this->list->id))->fetchOne();
if (!$q) {
$le = new LsListEntity();
$le->Entity = $entity;
$le->LsList = $this->list;
if (isset($match['rank'])) {
if (preg_match('/(\\d+)/isu', $match['rank'], $m)) {
$le->rank = $m[1];
}
}
$le->save();
$this->printDebug('List membership saved');
}
}
if ($this->org) {
$q = LsDoctrineQuery::create()->from('Relationship r')->where('r.entity1_id = ? and r.entity2_id = ? and r.category_id = ?', array($entity->id, $this->org->id, 1))->fetchOne();
if ($q) {
$this->printDebug(' (relationship already found, skipping...)');
return;
}
$relationship = new Relationship();
$relationship->Entity1 = $entity;
$relationship->Entity2 = $this->org;
$relationship->setCategory($this->relationship_category);
if ($this->description1) {
$relationship->description1 = $this->description1;
} else {
$description = $this->readline(" what description to give this relationship ({$relationship}) ? (less than 3 chars will skip)");
if (strlen($description) > 2) {
$relationship->description1 = $description;
}
}
if ($this->relationship_category == 'Position') {
$relationship->is_board = $this->is_board;
} else {
if ($this->relationship_category == 'Donation') {
if ($this->amount) {
$relationship->amount = $this->amount;
} else {
$amount = $this->readline(" what amount ({$relationship}) ? (less than 3 chars will skip)");
if (strlen($amount) > 1) {
$relationship->amount = $amount;
}
}
}
}
$relationship->save();
$relationship->addReference($this->url, null, null, $this->url_name);
$this->printDebug(" Relationship saved: {$relationship}");
}
}
//dump history
if (isset($match['affiliation1'])) {
$affiliation = $match['affiliation'];
//$this->printDebug($affiliation);
}
}
private function findBasicInfo()
{
if (!$this->sets) {
return null;
}
$re = '/^([^<]*?<[^>]*>)*?[^<]*?(?<!([\\.,$\\/]))(\\b[2-9]\\d\\b)(?!((,\\s+200\\d|199\\d)|%|[,\\.]\\d|[-\\s]+([Yy]ears?\\s+(with|career)|[Dd]ays?|[Mm]onths?)\\b))/su';
$age_match_sets = array();
//go through the sets of name matches and find age matches for each
foreach ($this->sets as $set) {
$age_matches = array();
for ($i = 0; $i < count($set); $i++) {
$len = $i == count($set) - 1 ? 2000 : $set[$i + 1]['pos'] - $set[$i]['pos'];
if ($len > 100000) {
continue;
}
$str = substr($this->text, $set[$i]['pos'], $len);
if (preg_match($re, $str, $match)) {
$n = preg_match_all('/<(\\p{L}+)[^>]*>/s', $match[0], $m, PREG_SET_ORDER);
$tag = 'empty';
if ($n > 0) {
$tag = $m[count($m) - 1][1];
}
$stripped = LsHtml::stripTags($match[0]);
if (strlen($stripped) < 2000) {
$age_matches[] = array('ind' => $i, 'age_match' => $match, 'age' => $match[3], 'name_match' => $set[$i], 'num_tags' => $n, 'tag' => $tag, 'len' => strlen($match[0]));
}
//$this->printDebug($i . '. ' . $set[$i]['name'] . ' : ' . $match[3] . ' : ' . strlen($match[0]) . ' : ' . $n . ' : ' . $tag);
//$this->printDebug($set[$i]['match'][1][0]);
}
//else $this->printDebug('--');
//$this->printDebug($set[$i]['match'][1][0]);
}
$this->printDebug('count age matches is ' . count($age_matches));
$age_match_sets[] = $age_matches;
}
//find the best set (most unique names and ages)
$max = 0;
$best = array(array('unique' => array(), 'set' => array()));
foreach ($age_match_sets as $age_matches) {
if (count($age_matches) < 2) {
continue;
}
$unique = array($age_matches[0]['name_match']['id']);
$temp = array($age_matches[0]);
for ($i = 1; $i < count($age_matches); $i++) {
if ($age_matches[$i]['ind'] - 4 <= $age_matches[$i - 1]['ind']) {
$temp[] = $age_matches[$i];
if (!in_array($age_matches[$i]['name_match']['id'], $unique)) {
$unique[] = $age_matches[$i]['name_match']['id'];
}
} else {
if (count($unique) > $max) {
$max = count($unique);
if (count(array_intersect($best[0]['unique'], $unique)) == 0 && count($best[0]['unique']) > 2) {
array_unshift($best, array('unique' => $unique, 'set' => $temp));
} else {
$best = array(array('unique' => $unique, 'set' => $temp));
}
} else {
if (count(array_intersect($best[0]['unique'], $unique)) == 0 && count($unique) > 2) {
$best[] = array('unique' => $unique, 'set' => $temp);
}
}
$unique = array($age_matches[$i]['name_match']['id']);
$temp = array($age_matches[$i]);
}
}
if (count($unique) > $max) {
$max = count($unique);
if (count(array_intersect($best[0]['unique'], $unique)) == 0) {
array_unshift($best, array('unique' => $unique, 'set' => $temp));
} else {
$best = array(array('unique' => $unique, 'set' => $temp));
}
}
}
$best = $best[0]['set'];
//$this->printDebug('count best is ' . count($best));
//find the tag all names have in common (if there is one)
$tag_counts = array();
foreach ($best as $b) {
if (isset($tag_counts[$b['tag']])) {
$tag_counts[$b['tag']]++;
} else {
$tag_counts[$b['tag']] = 1;
}
$this->printDebug($b['ind'] . '. ' . $b['name_match']['name'] . ' : ' . $b['age'] . ' : ' . strlen($b['age_match'][0]) . ' : ' . $b['num_tags'] . ' : ' . $b['tag']);
}
$tag = null;
foreach ($tag_counts as $k => $v) {
if ($v > 0.8 * count($best)) {
$tag = $k;
break;
}
}
$age_set = array();
if ($tag) {
foreach ($best as $b) {
if ($b['tag'] == $tag) {
$age_set[] = $b;
}
}
} else {
$age_set = $best;
}
$age_set = LsArray::multiSort($age_set, array('name_match', 'id'));
//find duplicates and determine the best match out of the pair/set
$singles = array();
$doubles = array();
$num_tags = 0;
$len = 0;
for ($i = 0; $i < count($age_set); $i++) {
$double = array($age_set[$i]);
while ($i < count($age_set) - 1 && $double[0]['name_match']['id'] == $age_set[$i + 1]['name_match']['id']) {
$double[] = $age_set[$i + 1];
$i++;
}
if (count($double) == 1) {
$singles[] = $age_set[$i];
$num_tags += $age_set[$i]['num_tags'];
$len += $age_set[$i]['len'];
} else {
$doubles[] = $double;
}
}
if (count($singles) < 3) {
$unique = array();
$sets = array(array());
$age_set = LsArray::multiSort($age_set, array('name_match', 'pos'));
foreach ($age_set as $a) {
//$this->printDebug($a['name_match']['name'] . ": ");
if (!in_array($a['name_match']['id'], $unique)) {
$unique[] = $a['name_match']['id'];
$sets[count($sets) - 1][] = $a;
} else {
$unique = array($a['name_match']['id']);
$sets[] = array($a);
}
}
$age_set = $sets[0];
} else {
$avg_len = $len / count($singles);
$avg_tags = $num_tags / count($singles);
//$this->printDebug('len is ' . $avg_len . ' and tags is ' . $avg_tags);
foreach ($doubles as $double) {
$best = null;
foreach ($double as $d) {
$lf = $d['len'] / $avg_len;
$tf = $d['num_tags'] / $avg_tags;
$f = abs(2 - ($lf + $tf));
if (!$best) {
$best = $d;
} else {
if (abs($avg_tags - $best['num_tags']) > abs($avg_tags - $d['num_tags'])) {
$best = $d;
} else {
if (abs($avg_tags - $best['num_tags']) == abs($avg_tags - $d['num_tags']) && abs($avg_len - $best['len']) == abs($avg_len - $d['len'])) {
$best = $d;
}
}
}
}
$singles[] = $best;
}
$age_set = LsArray::multiSort($singles, array('name_match', 'pos'));
}
//determine which directors were found, which weren't
$ids = array();
foreach ($age_set as $a) {
$ids[] = $a['name_match']['id'];
//$this->printDebug($a['ind'] . '. ' . $a['name_match']['name'] . ' : ' . $a['age'] . ' : ' . strlen($a['age_match'][0]) . ' : ' . $a['num_tags'] . ' : ' . $a['tag']);
}
foreach ($this->people as $p) {
if (!in_array($p->id, $ids)) {
$category = Doctrine::getTable('RelationshipCategory')->findOneByName('Position');
$relationship = LsDoctrineQuery::create()->from('Relationship r')->where('r.entity1_id = ?', $p->id)->addWhere('r.entity2_id = ?', $this->corp->id)->addWhere('r.category_id = ?', $category->id)->addWhere('r.description1 = ?', 'Director')->fetchOne();
if ($relationship) {
$relationship->is_current = 0;
$relationship->save();
}
}
}
if (count($age_set) < 0.5 * count($this->people)) {
$this->printDebug('not enough names in age set:' . count($age_set) . ' vs. ' . count($this->people));
return null;
}
//figure out which tags surround name/age pairs
$tag_arr = array('<table' => array(), '<tr' => array(), '<td' => array(), '<div' => array(), '<br' => array(), '<p' => array());
$tag_arr = array('table' => array(), 'tr' => array(), 'td' => array(), 'div' => array(), 'br' => array(), 'p' => array());
for ($i = 1; $i < count($age_set) - 1; $i++) {
$str = substr($this->text, $age_set[$i - 1]['name_match']['pos'], $age_set[$i + 1]['name_match']['pos'] - $age_set[$i - 1]['name_match']['pos']);
//$this->printDebug($str);
foreach ($tag_arr as $tag => &$arr) {
$tag_str = LsHtml::getStringInTag($str, $tag, $age_set[$i]['name_match']['pos'] - $age_set[$i - 1]['name_match']['pos']);
if (strlen($tag_str) > 0) {
$arr[] = strlen($tag_str);
//$this->printDebug($tag_str);
//echo "\n*****\n";
}
}
}
arsort($tag_arr);
//var_dump($tag_arr);
//$this->printDebug(count($this->people));
if (count(reset($tag_arr)) == 0) {
$this->printDebug('problems with enclosing tag detection');
return null;
}
foreach ($tag_arr as $tag => $arr) {
$avg = array_sum($arr) / count($arr);
$splitter = $tag;
break;
}
$tag_counts = array();
for ($i = 0; $i < count($age_set) - 1; $i++) {
$str = substr($this->text, $age_set[$i]['name_match']['pos'], $age_set[$i + 1]['name_match']['pos'] - $age_set[$i]['name_match']['pos']);
str_ireplace('<' . $splitter, ' ', $str, $count);
$tag_counts[] = $count;
}
sort($tag_counts);
$ct = $tag_counts[0];
if (!$ct) {
return null;
}
$post_strlen = 0;
$info_arr = array();
for ($i = 0; $i < count($age_set); $i++) {
$a = $age_set[$i];
$matches = LsString::striposMulti($this->text, '</' . $splitter, $ct, $a['name_match']['pos']);
$end = $matches[count($matches) - 1];
$start = strripos(substr($this->text, 0, $a['name_match']['pos']), '<' . $splitter);
$str = substr($this->text, $start, $end - $start);
if ($i == count($age_set) - 1 && count($matches) > 1) {
$end = $matches[count($matches) - 2];
$str2 = substr($this->text, $start, $end - $start);
$avg = strlen(implode(' ', $segments)) / count($segments);
if (abs(strlen($str2) - $avg) < abs(strlen($str) - $avg)) {
$str = $str2;
}
}
$segments[] = $str;
//$this->printDebug($str);
$info = $this->parseSegment($str, $a['name_match']['pos'] - $start, $a['name_match']['pos'] - $start + strlen($a['name_match']['match'][2][0]));
$info = $this->parseBlurb($info, $a);
//looks to see if bio appears aftr the parsed segment
if ($i < count($age_set) - 1) {
$next_start = strripos(substr($this->text, 0, $age_set[$i + 1]['name_match']['pos']), '<' . $splitter);
$post_str = substr($this->text, $end, $next_start - $end);
} else {
$avg = $post_strlen / (count($age_set) - 1);
$post_str = substr($this->text, $end, $avg);
}
$post_strlen += strlen($post_str);
$post_str = LsHtml::replaceFontStyleTags($post_str);
$person = $a['name_match']['person'];
$last = LsString::escapeStringForRegex($person->name_last);
$info['post_blurb'] = '';
if (preg_match_all('/>([^<]*' . $last . '[^<]*)</isu', $post_str, $matches)) {
$post_blurb = implode(' ', $matches[1]);
$post_blurb = trim(preg_replace('/\\s+/s', ' ', $post_blurb));
if (strlen($post_blurb) > 40) {
$info['post_blurb'] = $post_blurb;
}
}
$info_arr[] = $info;
//echo "\n\n***\n\n";
}
$ct = 0;
$unv_ct = 0;
foreach ($info_arr as $info) {
if (strlen($info['post_blurb']) > strlen($info['blurb'])) {
$ct++;
}
if ($info['img'] == null && $info['unverified_img'] != null) {
$unv_ct++;
}
}
//if most of the profile segments have images at the end, check to see if they belong to the next profile segment
if ($unv_ct > count($age_set) - 3) {
for ($i = 0; $i < count($age_set); $i++) {
$len = strripos(substr($this->text, 0, $age_set[$i]['name_match']['pos']), '<' . $splitter);
$tag_start = strripos(substr($this->text, 0, $len), '<img');
$str = substr($this->text, $tag_start, 200);
if (preg_match('/^<img[^>]+src=[\'"]([^\'"]+)[\'"]/is', $str, $match) == 1) {
$info['img'] = $match[1];
} else {
if ($i == 0) {
break;
}
}
}
}
for ($i = 0; $i < count($info_arr); $i++) {
if ($ct > 0.8 * count($age_set)) {
$info_arr[$i]['blurb'] = $info_arr[$i]['post_blurb'];
if (!$info_arr[$i]['since']) {
$info_arr[$i]['since'] = $this->getStartDate($info_arr[$i]['blurb']);
}
}
$this->importDirectorInfo($info_arr[$i], $age_set[$i]);
$this->printDebug("\n***");
}
//$this->printDebug($splitter);
//var_dump($tag_counts);
}
public function getCleanFirstParagraph()
{
if ($this->_paragraphs) {
return null;
}
$first = $this->_paragraphs[0];
$first = LsString::spacesToSpace(LsHtml::replaceEntities(LsHtml::stripTags($first)));
return $first;
}