private function findBasicInfo()
{
if (!$this->sets) {
return null;
}
$re = '/^([^<]*?<[^>]*>)*?[^<]*?(?<!([\\.,$\\/]))(\\b[2-9]\\d\\b)(?!((,\\s+200\\d|199\\d)|%|[,\\.]\\d|[-\\s]+([Yy]ears?\\s+(with|career)|[Dd]ays?|[Mm]onths?)\\b))/su';
$age_match_sets = array();
//go through the sets of name matches and find age matches for each
foreach ($this->sets as $set) {
$age_matches = array();
for ($i = 0; $i < count($set); $i++) {
$len = $i == count($set) - 1 ? 2000 : $set[$i + 1]['pos'] - $set[$i]['pos'];
if ($len > 100000) {
continue;
}
$str = substr($this->text, $set[$i]['pos'], $len);
if (preg_match($re, $str, $match)) {
$n = preg_match_all('/<(\\p{L}+)[^>]*>/s', $match[0], $m, PREG_SET_ORDER);
$tag = 'empty';
if ($n > 0) {
$tag = $m[count($m) - 1][1];
}
$stripped = LsHtml::stripTags($match[0]);
if (strlen($stripped) < 2000) {
$age_matches[] = array('ind' => $i, 'age_match' => $match, 'age' => $match[3], 'name_match' => $set[$i], 'num_tags' => $n, 'tag' => $tag, 'len' => strlen($match[0]));
}
//$this->printDebug($i . '. ' . $set[$i]['name'] . ' : ' . $match[3] . ' : ' . strlen($match[0]) . ' : ' . $n . ' : ' . $tag);
//$this->printDebug($set[$i]['match'][1][0]);
}
//else $this->printDebug('--');
//$this->printDebug($set[$i]['match'][1][0]);
}
$this->printDebug('count age matches is ' . count($age_matches));
$age_match_sets[] = $age_matches;
}
//find the best set (most unique names and ages)
$max = 0;
$best = array(array('unique' => array(), 'set' => array()));
foreach ($age_match_sets as $age_matches) {
if (count($age_matches) < 2) {
continue;
}
$unique = array($age_matches[0]['name_match']['id']);
$temp = array($age_matches[0]);
for ($i = 1; $i < count($age_matches); $i++) {
if ($age_matches[$i]['ind'] - 4 <= $age_matches[$i - 1]['ind']) {
$temp[] = $age_matches[$i];
if (!in_array($age_matches[$i]['name_match']['id'], $unique)) {
$unique[] = $age_matches[$i]['name_match']['id'];
}
} else {
if (count($unique) > $max) {
$max = count($unique);
if (count(array_intersect($best[0]['unique'], $unique)) == 0 && count($best[0]['unique']) > 2) {
array_unshift($best, array('unique' => $unique, 'set' => $temp));
} else {
$best = array(array('unique' => $unique, 'set' => $temp));
}
} else {
if (count(array_intersect($best[0]['unique'], $unique)) == 0 && count($unique) > 2) {
$best[] = array('unique' => $unique, 'set' => $temp);
}
}
$unique = array($age_matches[$i]['name_match']['id']);
$temp = array($age_matches[$i]);
}
}
if (count($unique) > $max) {
$max = count($unique);
if (count(array_intersect($best[0]['unique'], $unique)) == 0) {
array_unshift($best, array('unique' => $unique, 'set' => $temp));
} else {
$best = array(array('unique' => $unique, 'set' => $temp));
}
}
}
$best = $best[0]['set'];
//$this->printDebug('count best is ' . count($best));
//find the tag all names have in common (if there is one)
$tag_counts = array();
foreach ($best as $b) {
if (isset($tag_counts[$b['tag']])) {
$tag_counts[$b['tag']]++;
} else {
$tag_counts[$b['tag']] = 1;
}
$this->printDebug($b['ind'] . '. ' . $b['name_match']['name'] . ' : ' . $b['age'] . ' : ' . strlen($b['age_match'][0]) . ' : ' . $b['num_tags'] . ' : ' . $b['tag']);
}
$tag = null;
foreach ($tag_counts as $k => $v) {
if ($v > 0.8 * count($best)) {
$tag = $k;
break;
}
}
$age_set = array();
if ($tag) {
foreach ($best as $b) {
if ($b['tag'] == $tag) {
$age_set[] = $b;
}
}
} else {
$age_set = $best;
}
$age_set = LsArray::multiSort($age_set, array('name_match', 'id'));
//find duplicates and determine the best match out of the pair/set
$singles = array();
$doubles = array();
$num_tags = 0;
$len = 0;
for ($i = 0; $i < count($age_set); $i++) {
$double = array($age_set[$i]);
while ($i < count($age_set) - 1 && $double[0]['name_match']['id'] == $age_set[$i + 1]['name_match']['id']) {
$double[] = $age_set[$i + 1];
$i++;
}
if (count($double) == 1) {
$singles[] = $age_set[$i];
$num_tags += $age_set[$i]['num_tags'];
$len += $age_set[$i]['len'];
} else {
$doubles[] = $double;
}
}
if (count($singles) < 3) {
$unique = array();
$sets = array(array());
$age_set = LsArray::multiSort($age_set, array('name_match', 'pos'));
foreach ($age_set as $a) {
//$this->printDebug($a['name_match']['name'] . ": ");
if (!in_array($a['name_match']['id'], $unique)) {
$unique[] = $a['name_match']['id'];
$sets[count($sets) - 1][] = $a;
} else {
$unique = array($a['name_match']['id']);
$sets[] = array($a);
}
}
$age_set = $sets[0];
} else {
$avg_len = $len / count($singles);
$avg_tags = $num_tags / count($singles);
//$this->printDebug('len is ' . $avg_len . ' and tags is ' . $avg_tags);
foreach ($doubles as $double) {
$best = null;
foreach ($double as $d) {
$lf = $d['len'] / $avg_len;
$tf = $d['num_tags'] / $avg_tags;
$f = abs(2 - ($lf + $tf));
if (!$best) {
$best = $d;
} else {
if (abs($avg_tags - $best['num_tags']) > abs($avg_tags - $d['num_tags'])) {
$best = $d;
} else {
if (abs($avg_tags - $best['num_tags']) == abs($avg_tags - $d['num_tags']) && abs($avg_len - $best['len']) == abs($avg_len - $d['len'])) {
$best = $d;
}
}
}
}
$singles[] = $best;
}
$age_set = LsArray::multiSort($singles, array('name_match', 'pos'));
}
//determine which directors were found, which weren't
$ids = array();
foreach ($age_set as $a) {
$ids[] = $a['name_match']['id'];
//$this->printDebug($a['ind'] . '. ' . $a['name_match']['name'] . ' : ' . $a['age'] . ' : ' . strlen($a['age_match'][0]) . ' : ' . $a['num_tags'] . ' : ' . $a['tag']);
}
foreach ($this->people as $p) {
if (!in_array($p->id, $ids)) {
$category = Doctrine::getTable('RelationshipCategory')->findOneByName('Position');
$relationship = LsDoctrineQuery::create()->from('Relationship r')->where('r.entity1_id = ?', $p->id)->addWhere('r.entity2_id = ?', $this->corp->id)->addWhere('r.category_id = ?', $category->id)->addWhere('r.description1 = ?', 'Director')->fetchOne();
if ($relationship) {
$relationship->is_current = 0;
$relationship->save();
}
}
}
if (count($age_set) < 0.5 * count($this->people)) {
$this->printDebug('not enough names in age set:' . count($age_set) . ' vs. ' . count($this->people));
return null;
}
//figure out which tags surround name/age pairs
$tag_arr = array('<table' => array(), '<tr' => array(), '<td' => array(), '<div' => array(), '<br' => array(), '<p' => array());
$tag_arr = array('table' => array(), 'tr' => array(), 'td' => array(), 'div' => array(), 'br' => array(), 'p' => array());
for ($i = 1; $i < count($age_set) - 1; $i++) {
$str = substr($this->text, $age_set[$i - 1]['name_match']['pos'], $age_set[$i + 1]['name_match']['pos'] - $age_set[$i - 1]['name_match']['pos']);
//$this->printDebug($str);
foreach ($tag_arr as $tag => &$arr) {
$tag_str = LsHtml::getStringInTag($str, $tag, $age_set[$i]['name_match']['pos'] - $age_set[$i - 1]['name_match']['pos']);
if (strlen($tag_str) > 0) {
$arr[] = strlen($tag_str);
//$this->printDebug($tag_str);
//echo "\n*****\n";
}
}
}
arsort($tag_arr);
//var_dump($tag_arr);
//$this->printDebug(count($this->people));
if (count(reset($tag_arr)) == 0) {
$this->printDebug('problems with enclosing tag detection');
return null;
}
foreach ($tag_arr as $tag => $arr) {
$avg = array_sum($arr) / count($arr);
$splitter = $tag;
break;
}
$tag_counts = array();
for ($i = 0; $i < count($age_set) - 1; $i++) {
$str = substr($this->text, $age_set[$i]['name_match']['pos'], $age_set[$i + 1]['name_match']['pos'] - $age_set[$i]['name_match']['pos']);
str_ireplace('<' . $splitter, ' ', $str, $count);
$tag_counts[] = $count;
}
sort($tag_counts);
$ct = $tag_counts[0];
if (!$ct) {
return null;
}
$post_strlen = 0;
$info_arr = array();
for ($i = 0; $i < count($age_set); $i++) {
$a = $age_set[$i];
$matches = LsString::striposMulti($this->text, '</' . $splitter, $ct, $a['name_match']['pos']);
$end = $matches[count($matches) - 1];
$start = strripos(substr($this->text, 0, $a['name_match']['pos']), '<' . $splitter);
$str = substr($this->text, $start, $end - $start);
if ($i == count($age_set) - 1 && count($matches) > 1) {
$end = $matches[count($matches) - 2];
$str2 = substr($this->text, $start, $end - $start);
$avg = strlen(implode(' ', $segments)) / count($segments);
if (abs(strlen($str2) - $avg) < abs(strlen($str) - $avg)) {
$str = $str2;
}
}
$segments[] = $str;
//$this->printDebug($str);
$info = $this->parseSegment($str, $a['name_match']['pos'] - $start, $a['name_match']['pos'] - $start + strlen($a['name_match']['match'][2][0]));
$info = $this->parseBlurb($info, $a);
//looks to see if bio appears aftr the parsed segment
if ($i < count($age_set) - 1) {
$next_start = strripos(substr($this->text, 0, $age_set[$i + 1]['name_match']['pos']), '<' . $splitter);
$post_str = substr($this->text, $end, $next_start - $end);
} else {
$avg = $post_strlen / (count($age_set) - 1);
$post_str = substr($this->text, $end, $avg);
}
$post_strlen += strlen($post_str);
$post_str = LsHtml::replaceFontStyleTags($post_str);
$person = $a['name_match']['person'];
$last = LsString::escapeStringForRegex($person->name_last);
$info['post_blurb'] = '';
if (preg_match_all('/>([^<]*' . $last . '[^<]*)</isu', $post_str, $matches)) {
$post_blurb = implode(' ', $matches[1]);
$post_blurb = trim(preg_replace('/\\s+/s', ' ', $post_blurb));
if (strlen($post_blurb) > 40) {
$info['post_blurb'] = $post_blurb;
}
}
$info_arr[] = $info;
//echo "\n\n***\n\n";
}
$ct = 0;
$unv_ct = 0;
foreach ($info_arr as $info) {
if (strlen($info['post_blurb']) > strlen($info['blurb'])) {
$ct++;
}
if ($info['img'] == null && $info['unverified_img'] != null) {
$unv_ct++;
}
}
//if most of the profile segments have images at the end, check to see if they belong to the next profile segment
if ($unv_ct > count($age_set) - 3) {
for ($i = 0; $i < count($age_set); $i++) {
$len = strripos(substr($this->text, 0, $age_set[$i]['name_match']['pos']), '<' . $splitter);
$tag_start = strripos(substr($this->text, 0, $len), '<img');
$str = substr($this->text, $tag_start, 200);
if (preg_match('/^<img[^>]+src=[\'"]([^\'"]+)[\'"]/is', $str, $match) == 1) {
$info['img'] = $match[1];
} else {
if ($i == 0) {
break;
}
}
}
}
for ($i = 0; $i < count($info_arr); $i++) {
if ($ct > 0.8 * count($age_set)) {
$info_arr[$i]['blurb'] = $info_arr[$i]['post_blurb'];
if (!$info_arr[$i]['since']) {
$info_arr[$i]['since'] = $this->getStartDate($info_arr[$i]['blurb']);
}
}
$this->importDirectorInfo($info_arr[$i], $age_set[$i]);
$this->printDebug("\n***");
}
//$this->printDebug($splitter);
//var_dump($tag_counts);
}
