protected function execute($arguments = array(), $options = array()) { $configuration = ProjectConfiguration::getApplicationConfiguration($options['application'], $options['env'], true); $databaseManager = new sfDatabaseManager($configuration); $databaseManager->initialize($configuration); $q = EntityTable::getByExtensionQuery(array('Person', 'ElectedRepresentative'))->addWhere('summary like ? OR summary like ? OR summary like ? OR summary like ? OR summary like ? OR summary like ? OR summary like ? OR summary like ? OR summary like ?', array('(daughter%', '(son%', '(father%', '(mother%', '(cousin%', '(husband%', '(wife%', '(brother%', '(sister%'))->orderBy('person.name_last'); $members = $q->execute(); foreach ($members as $member) { if (preg_match('/\\([^\\)]*\\)/isu', $member->summary, $match)) { echo $member->name . ":\n"; if (preg_match_all('/(brother|sister|daughter|mother|father|wife|husband|cousin)\\sof\\s+([^\\;\\)\\,]*)(\\;|\\)|\\,)/isu', $match[0], $matches, PREG_SET_ORDER)) { foreach ($matches as $m) { echo "\t\t" . $m[1] . ' : of : ' . $m[2] . "\n"; $m[2] = str_replace('.', '', $m[2]); $parts = LsString::split($m[2]); $q = EntityTable::getByExtensionQuery(array('Person', 'ElectedRepresentative')); foreach ($parts as $part) { $q->addWhere('e.name like ?', '%' . $part . '%'); } $people = $q->execute(); $family = array(); foreach ($people as $person) { echo "\t\t\t\t" . $person->name . "\n"; if ($person->id != $member->id) { $family[] = $person; } } if (count($family) == 1) { $q = LsDoctrineQuery::create()->from('Relationship r')->where('(r.entity1_id = ? or r.entity2_id =?) and (r.entity1_id = ? or r.entity2_id = ?)', array($member->id, $member->id, $person->id, $person->id)); if (!$q->count()) { if ($description2 = FamilyTable::getDescription2($m[1], $family[0]->Gender->id)) { $relationship = new Relationship(); $relationship->setCategory('Family'); $relationship->Entity1 = $member; $relationship->Entity2 = $person; $relationship->description1 = $m[1]; $relationship->description2 = $description2; $relationship->save(); $ref = LsQuery::getByModelAndFieldsQuery('Reference', array('object_model' => 'Entity', 'object_id' => $member->id, 'name' => 'Congressional Biographical Directory'))->fetchOne(); if ($ref) { $relationship->addReference($ref->source, null, null, $ref->name, $ref->source_detail, $ref->publication_date); } echo "-------------------------------added relationship\n"; } } } } } echo "\n"; } } }
private function updateCommitteeDetails(Entity $committee) { $this->printDebug(" Updating committee details: " . $this->fecCommitteeUrl . $committee->fec_id); if (!$this->browser->get($this->fecCommitteeUrl . $committee->fec_id)->responseIsError()) { $text = $this->browser->getResponseText(); if (preg_match('/Treasurer Name:<\\/B><\\/TD><TD>(.+)<\\/TD><\\/TR>/', $text, $treasurer_name)) { //nothing } $committee_designation = null; if (preg_match('/Committee Designation: <\\/B><\\/TD><TD>(\\w)/', $text, $committee_designation)) { switch (trim($committee_designation[1])) { case 'P': $committee->addExtension('IndividualCampaignCommittee'); $committee_designation = "Principal Campaign Committee"; break; case 'A': $committee->addExtension('IndividualCampaignCommittee'); $committee_designation = "Authorized Campaign Committee"; break; default: $committee->addExtension('OtherCampaignCommittee'); $committee_designation = "Other Campaign Committee"; } } else { $committee->addExtension('OtherCampaignCommittee'); $committee_designation = "Other Campaign Committee"; } if (preg_match('/Committee Type: <\\/B><\\/TD><TD WIDTH=300>([^<]+)<\\/TD><\\/TR>/', $text, $committee_type)) { //var_dump($committee_type); $type = null; switch (trim($committee_type[1])) { case 'PRESIDENTIAL': $type = 'pres'; break; case 'HOUSE': $type = 'house'; break; case 'SENATE': $type = 'senate'; break; } } if (preg_match('/Candidate State:<\\/B><\\/TD><TD>([^<]+)<\\/TD><\\/TR>/', $text, $candidate_state)) { //var_dump($candidate_state); if (trim($candidate_state[1]) != "Presidential Candidate") { $committee->State = AddressStateTable::retrieveByText($candidate_state[1]); } } if (preg_match_all('#cgi-bin/can_detail/([^"]+)?\\/">([^<]+)</A>#i', $text, $candidates, PREG_PATTERN_ORDER)) { /* * $candidates[1] = FEC ID * $candidates[2] = NAME */ foreach ($candidates[1] as $key => $candidate_id) { $candidate_name = $candidates[2][$key]; $found_candidate = null; //look for candidate by their FEC IDs $found_candidate = $this->getCandidatesQuery()->addWhere('politicalcandidate.senate_fec_id = ? OR politicalcandidate.house_fec_id = ? OR politicalcandidate.pres_fec_id = ?', array($candidate_id, $candidate_id, $candidate_id))->fetchOne(); $found_msg = $found_candidate ? " FOUND" : " NOT FOUND"; $this->printDebug(" + Looking for candidate by ID: " . $found_msg); //next look for them by their names $where_clause = null; $p1 = null; if (!$found_candidate) { $p1 = PersonTable::parseCommaName($candidate_name); $candidate_query = $this->getCandidatesQuery()->addWhere("person.name_first = ? AND person.name_last = ?", array($p1->name_first, $p1->name_last)); if (strlen($p1->name_middle)) { $candidate_query = $candidate_query->addWhere('person.name_middle LIKE ?', "%" . $p1->name_middle); } $found_candidate = $candidate_query->fetchOne(); $found_msg = $found_candidate ? " FOUND" : " NOT FOUND"; $this->printDebug(" + Looking for candidate by name: " . $p1->name_first . " " . $p1->name_last . $found_msg . " "); //candidate is not in database. we should add them if (!$found_candidate) { if (!$this->browser->get($this->fecCandidateUrl . $candidate_id)->responseIsError()) { $text = $this->browser->getResponseText(); $office_sought_arr = $this->getOfficeSought($text); if (!$office_sought_arr[1][0]) { continue; } $office_sought = $office_sought_arr[1][0]; $found_candidate = PersonTable::parseCommaName($candidate_name); $found_candidate->addExtension('PoliticalCandidate'); $found_candidate->is_federal = true; $found_candidate->is_state = false; $found_candidate->is_local = false; $found_candidate->pres_fec_id = $office_sought == 'President' ? $candidate_id : null; $found_candidate->senate_fec_id = $office_sought == 'Senate' ? $candidate_id : null; $found_candidate->house_fec_id = $office_sought == 'House' ? $candidate_id : null; $found_candidate->save(); $found_candidate->addReference($source = $this->fecCandidateUrl . $candidate_id, $excerpt = null, $fields = array('name_first', 'name_middle', 'name_last', 'name_prefix', 'name_suffix', 'pres_fec_id', 'senate_fec_id', 'house_fec_id'), $name = 'FEC Disclosure Report', $detail = null, $date = null, false); $this->printDebug(" Adding new candidate: " . $p1->name_first . " " . $p1->name_middle . " " . $p1->name_last); } } } $q = LsDoctrineQuery::create()->from('Relationship r')->where('r.entity1_id = ? AND r.entity2_id = ? AND r.category_id = ? AND description1 = ?', array($committee->id, $found_candidate->id, RelationshipTable::DONATION_CATEGORY, $committee_designation)); if (!$q->count()) { $relationship = new Relationship(); $relationship->Entity1 = $committee; $relationship->Entity2 = $found_candidate; $relationship->setCategory('Donation'); //$relationship->amount = $donation_amount; $relationship->description1 = $committee_designation; //$relationship->fec_donation_id = $donation_fec_id; //$relationship->start_date = $donation_date; //$relationship->end_date = $donation_date; $relationship->is_current = 1; $relationship->save(); //campaign contribution $relationship->addReference($source = $this->fecCandidateUrl . $candidate_id, $excerpt = null, $fields = array('description1'), $name = 'FEC Disclosure Report', $detail = null, $date = null); } } } } else { $this->printDebug("Couldn't get " . $this->fecCommitteeUrl); } $committee->save(); $this->printDebug(" Updating committee details: COMPLETE"); }
protected function import($url) { $person = null; $this->printDebug($url); if (!$this->browser->get($url)->responseIsError()) { $text = $this->browser->getResponseText(); $bio = null; $name = null; $netWorth = null; $birthYear = null; $schools = null; $schools = null; $imageUrl = null; $rank = null; //get name & rank if ($this->year > 2005 && preg_match('/<b>#(\\d+) ([^<]+)<\\/b>/', $text, $match)) { $name = trim($match[2]); $rank = $match[1]; } if ($this->year == 2005 && preg_match('/<h2>#(\\d+) ([^<]+)<\\/h2>/', $text, $match)) { $name = trim($match[2]); $rank = $match[1]; } //get net worth if (preg_match('/Net Worth<\\/span> <span class="red">\\$([\\S]+) billion/', $text, $match)) { $netWorth = $match[1] * 1000000000; } //get birth year if (preg_match('/>Age<\\/span> (\\d+)/', $text, $match)) { $birthYear = date("Y") - $match[1] . "-00-00"; } //get schools if (preg_match('/Education<\\/span>(.*)<\\/td>/isU', $text, $match)) { $schools = array(); $schoolParts = explode('<br>', $match[1]); while ($schoolPart = current($schoolParts)) { if (preg_match('/^([^,]+),\\s+<b>([^<]+)<\\/b>/is', trim($schoolPart), $match)) { $schoolOrg = trim($match[1]); if ($schoolOrg == 'High School') { next($schoolParts); continue; } $schoolDegree = trim($match[2]); $schools[] = array('org' => $schoolOrg, 'degree' => $schoolDegree); } next($schoolParts); } } if (preg_match('#<br>[\\n\\s]<br>(.+?)<br>[\\n\\s]<br>[\\n\\s]<img#isU', $text, $match)) { $bio = strip_tags(trim($match[1])); } else { $wikipedia = new LsWikipedia(); if ($wikipedia->request($name)) { $bio = $wikipedia->getIntroduction(); } } //get image $regexp = '#([A-Z1-9]{4}).html#'; if (preg_match($regexp, $url, $match)) { $imageFilename = $match[1] . ".jpg"; $imageUrl = $this->list_urls[$this->year]['img_src'] . $imageFilename; } //echo "Rank: " . $rank . "\n"; $this->printDebug("Rank: " . $rank); $this->printDebug("Name: " . $name); $this->printDebug("Image: " . $imageUrl); $this->printDebug("Net worth: " . $netWorth); $this->printDebug("Birth year: " . $birthYear); $this->printDebug("Bio: " . $bio); $person = $this->generatePerson($name, $bio); $person_exists = $this->getBusinessPersonQuery()->addWhere("person.name_first = ? AND person.name_last = ?", array($person->name_first, $person->name_last))->fetchOne(); if ($person_exists != false) { $this->printDebug('Person exists'); $person = $person_exists; } else { $this->printDebug('Saving new person'); } //parse name and create person object $person->addExtension('BusinessPerson'); $person->start_date = $person->start_date == null ? $birthYear : $person->start_date; $person->summary = $person->summary == null ? $bio : $person->summary; $person->net_worth = $person->net_worth == null ? $netWorth : $person->net_worth; //go through schools person attended foreach ($schools as $school) { //does the current school exist? $current_school = EntityTable::getByExtensionQuery('Org')->addWhere("org.name = ?", $school['org'])->fetchOne(); if ($current_school) { $this->printDebug(" Found School " . $school['org']); } else { //clear cache Doctrine::getTable('ExtensionDefinition')->clear(); $current_school = new Entity(); $current_school->addExtension('Org'); $current_school->addExtension('School'); $current_school->name = LsLanguage::titleize($school['org']); $current_school->save(); $current_school->addReference($source = $url, $excerpt = null, $fields = array('name'), $name = 'Forbes.com', $detail = null, $date = null); $this->printDebug(" Adding new school: " . $school['org']); } //if there is no relationship between person and school. connect them! if (!$person->getRelationshipsWithQuery($current_school, RelationshipTable::EDUCATION_CATEGORY)->fetchOne()) { $this->printDebug(" Creating Relation between " . $current_school->name . " and " . $person->name); $education = new Relationship(); $education->Entity1 = $person; $education->Entity2 = $current_school; $education->setCategory('Education'); $education->description1 = $school['degree']; $education->is_current = 1; $education->save(); $education->addReference($source = $url, $excerpt = null, $fields = array('description1'), $name = 'Forbes.com', $detail = null, $date = null); } } $person->save(); $person->addReference($source = $url, $excerpt = null, $fields = array('name_prefix', 'name_first', 'name_middle', 'name_last', 'name_suffix', 'name_nick', 'summary', 'net_worth', 'start_date'), $name = 'Forbes.com', $detail = null, $date = null); $this->saveToList($person, $rank); $this->attachImage($person, $imageUrl); } else { echo "Couldn't get person: " . $url . "\n"; } }
protected function importGovernor($row) { $url = $this->_baseUrl . $row['url']; if (!$this->browser->get($url)->responseIsError()) { $text = $this->browser->getResponseText(); $text = LsHtml::replaceEntities($text); //preg_match('/>Family\:<\/b>([^<]*)<br/is',$text,$family_arr); $name = trim(str_ireplace('Gov.', '', $row['name'])); $this->printDebug(''); $this->printDebug($name . ':'); $governor = PersonTable::parseFlatName($name); $governor->addExtension('PoliticalCandidate'); $governor->addExtension('ElectedRepresentative'); $governor->is_state = 1; $similar = $governor->getSimilarEntitiesQuery(true)->execute(); foreach ($similar as $s) { $sim_re = LsString::escapeStringForRegex($s->name_first); $search_re = LsString::escapeStringForRegex($governor->name_first); if (preg_match('/^' . $sim_re . '/su', $governor->name_first) == 0 && preg_match('/^' . $search_re . '/su', $s->name_first) == 0) { continue; } $bio = $s->getExtendedBio(); if (preg_match('/\\bgovernor(ship)?\\b/isu', $bio)) { $governor = $s; $this->printDebug(' Found existing governor: ' . $s->name . ' ' . $s->id); break; } } $governor->save(); $this->printDebug($governor->id); if (!$governor->start_date && preg_match('/>Born\\:<\\/b>([^<]*)<br/is', $text, $birth_arr)) { $this->printDebug(' Birthdate: ' . $birth_arr[1]); $governor->start_date = trim($birth_arr[1]); } if (!$governor->birthplace && preg_match('/>Birth State\\:<\\/b>([^<]*)<br/is', $text, $birth_state_arr)) { $this->printDebug(' Birthplace: ' . trim($birth_state_arr[1])); $governor->birthplace = trim($birth_state_arr[1]); } //PARTY MEMBERSHIP if (preg_match('/>Party\\:<\\/b>([^<]*)<br/is', $text, $party_arr)) { $party_str = $party_arr[1]; $this->printDebug(' Party: ' . $party_str); if (stristr($party_str, 'Democrat')) { $party = EntityTable::getByExtensionQuery('PoliticalParty')->addWhere('name = ?', 'Democratic Party')->fetchOne(); } if (stristr($party_str, 'Republican')) { $party = EntityTable::getByExtensionQuery('PoliticalParty')->addWhere('name = ?', 'Republican Party')->fetchOne(); } if (isset($party) && $party && !$governor->party_id) { $governor->Party = $party; $governor->is_independent = false; $this->printDebug(' Added membership in ' . $party); } else { if (stristr($party_str, 'Independent')) { $governor->is_independent = true; } } } if (!$governor->summary && preg_match_all('/>([^<]{240,})/isu', $text, $bio_match)) { $str = ''; foreach ($bio_match[1] as $b) { if (!stristr($b, 'Javascript')) { $str .= "\n\n" . $b; } } $str = trim($str); if (strlen($str)) { $governor->summary = $str; } } $governor->save(); $governor->addReference($url, null, $governor->getAllModifiedFields(), 'Governors Association'); //SCHOOLS if (preg_match('/>School\\(s\\)\\:<\\/b>([^<]*)<br/is', $text, $school_arr)) { $school_names = explode(';', trim($school_arr[1])); if (count($school_names) == 1) { $school_names = explode(',', $school_names[0]); } foreach ($school_names as $school_name) { $school_name = trim($school_name); if (!($school = EntityTable::getByExtensionQuery('School')->leftJoin('e.Alias a')->addWhere('e.name = ? or a.name = ?', array($school_name, $school_name))->fetchOne())) { $school = new Entity(); $school->addExtension('Org'); $school->addExtension('School'); $school->name = $school_name; $school->save(); $this->printDebug(' Added School: ' . $school_name); } $q = RelationshipTable::getByCategoryQuery('Education')->addWhere('entity1_id = ? and entity2_id = ?', array($governor->id, $school->id))->fetchOne(); if (!$q) { $relationship = new Relationship(); $relationship->setCategory('Education'); $relationship->Entity1 = $governor; $relationship->Entity2 = $school; $relationship->is_current = 0; $relationship->save(); $relationship->addReference($url, null, $relationship->getAllModifiedFields(), 'Governors Association'); $this->printDebug(' Added education: ' . $relationship->name); } } } //GOVERNOR OFFICE AND POSITION $office_name = 'Office of the Governor of ' . $row['state']; if (!($office = EntityTable::getByExtensionQuery('GovernmentBody')->addWhere('name = ?', $office_name)->fetchOne())) { $office = new Entity(); $office->name = $office_name; $office->addExtension('Org'); $office->addExtension('GovernmentBody'); $state = Doctrine::getTable('AddressState')->findOneByName($row['state']); if ($state) { $office->state_id = $state->id; } $office->save(); $office->addReference($url, null, $office->getAllModifiedFields(), 'Governors Association'); $this->printDebug(' Added office: ' . $office->name); } $q = RelationshipTable::getByCategoryQuery('Position')->addWhere('entity1_id = ? and entity2_id = ? and description1 = ?', array($governor->id, $office->id, 'Governor'))->fetchOne(); if (!$q) { sort($row['years']); $i = 0; while ($i < count($row['years'])) { $governorship = new Relationship(); $governorship->setCategory('Position'); $governorship->Entity1 = $governor; $governorship->Entity2 = $office; $governorship->description1 = 'Governor'; $governorship->start_date = $row['years'][$i]; $i++; if (isset($row['years'][$i])) { $governorship->end_date = $row['years'][$i]; $governorship->is_current = 0; if (!$governor->blurb && !isset($row['years'][$i + 1])) { $governor->blurb = 'Former Governor of ' . $row['state']; } } else { $governorship->is_current = 1; if (!$governor->blurb) { $governor->blurb = 'Governor of ' . $row['state']; } } $governor->save(); $i++; $governorship->save(); $governorship->addReference($url, null, $governorship->getAllModifiedFields(), 'Governors Association'); $this->printDebug(' Added governorship: ' . $governorship->name); } } //SPOUSE if (preg_match('/>Spouse\\:<\\/b>(.*?)<br/is', $text, $spouse_arr)) { $spouse = trim(LsHtml::stripTags($spouse_arr[1])); $q = RelationshipTable::getByCategoryQuery('Family')->addWhere('entity1_id = ? or entity2_id = ?', array($governor->id, $governor->id))->fetchOne(); if (!$q && strlen($spouse)) { $spouse = PersonTable::parseFlatName($spouse); $spouse->save(); $this->printDebug(' Added spouse: ' . $spouse->name); $relationship = new Relationship(); $relationship->setCategory('Family'); $relationship->Entity1 = $spouse; $relationship->Entity2 = $governor; $relationship->description1 = 'Spouse'; $relationship->description2 = 'Spouse'; $relationship->save(); $relationship->addReference($url, null, $relationship->getAllModifiedFields(), 'Governors Association'); $this->printDebug(' Added spouse relationship: ' . $relationship->name); } } //ADDRESS --not working, malformed addresses /* if (preg_match('/>Address\:\s*<\/b>(.*?)<b>/is',$text,$address_arr)) { $address = trim(str_replace('<br/>',', ',$address_arr[1])); $this->printDebug($address); if ($governor->Address->count() == 0 && $a = $governor->addAddress($address)) { $this->printDebug(' Address: ' . $a); $governor->save(); } }*/ //PHONE NUMBER if (preg_match('/>Phone\\(s\\)\\:<\\/b>([^<]*)<br/is', $text, $phone_arr)) { $phone_number = trim($phone_arr[1]); if (!$governor->Phone->count()) { $phone = $governor->addPhone($phone_number); $this->printDebug(' Phone: ' . $phone); } } if (!$governor->Image->count() && preg_match('/<img .*?class\\="display" src\\="([^"]*)"/is', $text, $img_arr)) { $url = $img_arr[1]; try { $fileName = ImageTable::createFiles($url, $governor->name_first); } catch (Exception $e) { $fileName = null; } if ($fileName) { //insert image record $image = new Image(); $image->filename = $fileName; $image->entity_id = $governor->id; $image->title = $governor->name; $image->caption = 'From Governors Association website'; $image->is_featured = true; $image->is_free = false; $image->url = $url; $image->save(); $this->printDebug("Imported image: " . $image->filename); } } } }
function import(Entity $person, $possible_persons) { //loop through the people we found. usually just one. foreach ($possible_persons as $possible_person) { $this->printDebug('Query returned ' . count($possible_person) . ' person named ' . $possible_person->name); //this person does not provide education. we skip if (count($possible_person->education)) { $this->printDebug('Education found'); } else { $this->printDebug('No education history found'); continue; } //get employement info for this possible match $possible_person_bio = $possible_person->summary; if (count($possible_person->employment_history)) { foreach ($possible_person->employment_history as $employment) { $possible_person_bio .= ' ' . $employment->company . " "; } $this->printDebug('Employment found'); } else { $this->printDebug('No employment history found'); continue; } //get employment info for the person in our database $relationship_orgs = $person->getRelatedEntitiesQuery('Org', RelationshipTable::POSITION_CATEGORY, null, null, null, false, 1)->execute(); $person_bio = $person->summary; foreach ($relationship_orgs as $org) { $person_bio .= ' ' . $org->name; } //lets see how many matches we get $matches = LsLanguage::getCommonPronouns($person_bio, trim($possible_person_bio), LsLanguage::$business); if (count($matches)) { foreach ($possible_person->education as $school) { $school->institution = mb_convert_encoding($school->institution, 'UTF-8'); $school->institution = preg_replace('//isu', ' ', $school->institution); $this->printDebug('Looking for the school: ' . $school->institution); $current_school = EntityTable::findByAlias($school->institution, $context = 'bw_school'); //find school if ($current_school) { $this->printDebug('Found school'); } else { $current_school = EntityTable::getByExtensionQuery(array('Org', 'School'))->addWhere('LOWER(org.name) LIKE ?', '%' . strtolower($school->institution) . "%")->fetchOne(); if (!$current_school) { $new_school = new Entity(); $new_school->addExtension('Org'); $new_school->addExtension('School'); $new_school->name = $school->institution; $wikipedia = new LsWikipedia(); $wikipedia->request($school->institution); if ($wikipedia->execute() && !$wikipedia->isDisambiguation()) { $info_box = $wikipedia->getInfoBox(); if (isset($info_box['students']) && preg_match('/([\\d\\,]{2,})/isu', $info_box['students']['clean'], $match)) { $new_school->students = LsNumber::clean($match[1]); } else { $student_types = array('undergrad', 'postgrad', 'grad', 'doctoral'); $num_students = 0; foreach ($student_types as $st) { if (isset($info_box[$st]) && preg_match('/([\\d\\,]{2,})/isu', $info_box[$st]['clean'], $match)) { $num_students += LsNumber::clean($match[1]); } } if ($num_students > 0) { $new_school->students = $num_students; } } if (isset($info_box['faculty']) && preg_match('/([\\d\\,]{2,})/isu', $info_box['faculty']['clean'], $match)) { $new_school->faculty = LsNumber::clean($match[1]); } if (isset($info_box['type'])) { if (stristr($info_box['type']['clean'], 'public')) { $new_school->is_private = 0; } else { if (stristr($info_box['type']['clean'], 'private')) { $new_school->is_private = 1; } } } if (isset($info_box['endowment'])) { if (preg_match('/(\\$[\\d\\,\\.\\s]+)(million|billion)/isu', $info_box['endowment']['clean'], $match)) { if (strtolower($match[2]) == 'billion') { $factor = 1000000000; } else { $factor = 1000000; } $new_school->endowment = LsNumber::formatDollarAmountAsNumber($match[1], $factor); } } if (isset($info_box['established'])) { $year = null; if ($date = LsDate::convertDate($info_box['established']['clean'])) { $new_school->start_date = $date; } else { if (preg_match('/\\b(\\d\\d\\d\\d)\\b/isu', $info_box['established']['clean'], $match)) { $new_school->start_date = $match[1]; } } } $summary = trim($wikipedia->getIntroduction()); $summary = preg_replace('/\\n\\s*\\n/isu', '', $summary); if (strlen($summary) > 10) { $new_school->summary = $summary; } $new_school->save(); $new_school->addReference($source = $wikipedia->getUrl(), $excerpt = null, $fields = array('summary'), $name = 'Wikipedia'); } else { $new_school->save(); } $current_school = $new_school; $this->printDebug('Adding new school'); } $alias = new Alias(); $alias->name = $school->institution; $alias->context = 'bw_school'; $alias->Entity = $current_school; $alias->save(); } //find degree $degree = null; if (!($degree = DegreeTable::getByText($school->degree))) { $degree = DegreeTable::addDegree($school->degree); $this->printDebug('Adding new degree'); } //find relationship $relationship = null; $relationships = $person->getRelationshipsWithQuery($current_school, RelationshipTable::EDUCATION_CATEGORY)->execute(); foreach ($relationships as $existing_relationship) { if ($existing_relationship->degree_id == $degree->id) { $relationship = $existing_relationship; break; } } if ($relationship) { $this->printDebug('Relationship between person and school exists'); } else { $relationship = new Relationship(); $relationship->Entity1 = $person; $relationship->Entity2 = $current_school; $relationship->description1 = 'student'; $relationship->is_current = 0; if ($school->year) { $relationship->end_date = $school->year; } $relationship->setCategory('Education'); $this->printDebug('Creating new relationship between person and school'); } //save $relationship->save(); //add degree and reference if ($relationship->degree_id == null) { $reference_name = strstr($school->source, 'wikipedia') ? "Wikipedia" : "BusinessWeek"; $relationship->Degree = $degree; $relationship->save(); $relationship->addReference($source = $school->source, $excerpt = null, $fields = array('degree_id'), $name = $reference_name, $detail = null, $date = null); $this->printDebug('Adding degree and reference'); } } } else { $this->printDebug('No organization matches'); return false; } } return true; }
public function processCongressMemberRow($row) { $this->_references['bioguide'] = new Reference(); $this->printDebug("\nProcessing member with name " . $row->name . " and ID " . $row->id); try { //we have to begin the transaction here because matchInDatabase might merge Entities and save $this->db->beginTransaction(); //check that congress member isn't a repeat $member = EntityTable::getByExtensionQuery('ElectedRepresentative')->addWhere('electedrepresentative.bioguide_id = ?', $row->id)->fetchOne(); //if member hasn't been imported already as a member of congress, //create with all bio info and look for a match if (!$member) { $member = $this->importNewMember($row); $member = $this->matchInDatabase($member); /* if(!$merged_member = $this->matchInDatabase($member)) { $redundant_member = false; } else { $redundant_member = $member; $member = $merged_member; } */ } else { $this->printDebug("Member exists in database with entity ID: " . $member->id); //if member is tagged with this session, skip if (in_array($member->id, $this->_existingSessionMemberIds)) { $this->db->rollback(); $this->printDebug("Member has already been tagged with session " . current($this->_sessions) . "; skipping"); return $member; } //update member's bio $this->updateBio($member); $this->printDebug("Updated member bio"); //check if member is continuing from previous session $q = $member->getTripleTagsQuery('congress', 'session')->addWhere('tag.triple_value = ?', current($this->_sessions) - 1); if ($q->count()) { $this->printDebug("Continuing member from previous session..."); //if member continuing, look for relationship with opposite chamber to end, //in case the member's switched chambers $oppositeChamberId = $row->type == 'Senator' ? $this->_houseEntityId : $this->_senateEntityId; $q = LsDoctrineQuery::create()->from('Relationship r')->where('r.entity1_id = ? AND r.entity2_id = ?', array($member->id, $oppositeChamberId))->andWhere('r.category_id = ? AND r.end_date IS NULL', RelationshipTable::MEMBERSHIP_CATEGORY); foreach ($q->execute() as $rel) { $rel->end_date = $this->_sessionStartYear - 1 . '-00-00'; $rel->is_current = false; $rel->save(); $this->printDebug("Ended relationship " . $rel->id . " with opposite chamber from previous session"); } //if no current relationships with same chamber, create one $thisChamberId = $row->type == 'Senator' ? $this->_senateEntityId : $this->_houseEntityId; $q = LsDoctrineQuery::create()->from('Relationship r')->where('r.entity1_id = ? AND r.entity2_id = ?', array($member->id, $thisChamberId))->andWhere('r.category_id = ? AND r.end_date IS NULL', RelationshipTable::MEMBERSHIP_CATEGORY); if (!$q->count()) { $this->printDebug("No relationships with this chamber from previous session; creating new one..."); $r = new Relationship(); $r->entity1_id = $member->id; $r->entity2_id = $row->type == 'Senator' ? $this->_senateEntityId : $this->_houseEntityId; $r->setCategory('Membership'); $r->description1 = $row->type; $r->start_date = $row->termStart . '-00-00'; $r->is_current = true; if ($row->type = 'Senator') { $this->_senateRelationships[] = $r; } else { $this->_houseRelationships[] = $r; } } } else { $this->printDebug("Member not continuing from previous session; creating new relationship..."); //if member not continuing, add a new relationship for this session and chamber $r = new Relationship(); $r->entity1_id = $member->id; $r->entity2_id = $row->type == 'Senator' ? $this->_senateEntityId : $this->_houseEntityId; $r->setCategory('Membership'); $r->description1 = $row->type; $r->start_date = $row->termStart . '-00-00'; $r->is_current = true; if ($row->type = 'Senator') { $this->_senateRelationships[] = $r; } else { $this->_houseRelationships[] = $r; } } } //set party name $partyName = null; $blurb = ''; if ($party = $row->party) { if ($party == 'Democrat') { $blurb .= $party . 'ic'; $partyName = 'Democratic Party'; } elseif ($party == 'Independent') { $blurb .= $party; $partyName = null; } else { $blurb .= $party; $partyName = $party . ' Party'; } //if party entity doesn't exist, create one if ($partyName) { $q = EntityTable::getByExtensionQuery('PoliticalParty')->addWhere('e.name = ?', $partyName); if (!($partyEntity = $q->fetchOne())) { $partyEntity = new Entity(); $partyEntity->addExtension('Org'); $partyEntity->addExtension('PoliticalParty'); $partyEntity->name = $partyName; $partyEntity->save(); $this->printDebug("Created new political party: " . $partyName); } } //create current party affiliation if session is member's most recent session if ($member->exists()) { $q = $member->getTripleTagsQuery('congress', 'session')->addWhere('tag.triple_value > ?', current($this->_sessions)); $setParty = $q->count() ? false : true; } else { $setParty = true; } if ($setParty) { if ($partyName) { $member->Party = $partyEntity; $member->is_independent = false; $this->printDebug("Set current political affiliation to " . $partyName); } else { $member->is_independent = true; $member->party_id = null; $this->printDebug("Set current political affiliation to Independent"); } } } //save member $modified = $member->getAllModifiedFields(); $member->save(); $this->printDebug("Saved member with entity ID: " . $member->id); $this->addListMember($member); //set member reference fields $excludeFields = array(); foreach ($this->_references as $key => $ref) { $ref->object_model = 'Entity'; $ref->object_id = $member->id; if ($key != 'bioguide') { $ref->save(); $excludeFields = array_merge($excludeFields, $ref->getFieldsArray()); } } $modified = array_diff($modified, $excludeFields); $this->_references['bioguide']->addFields($modified); $this->_references['bioguide']->save(); $this->printDebug("Saved member references"); //tag member with congress session $member->addTagByTriple('congress', 'session', current($this->_sessions)); $this->printDebug("Added tag for session " . current($this->_sessions)); //save image, if any if ($this->_image) { $this->_image->Entity = $member; $this->_image->save(); $this->printDebug("Saved member image"); if ($this->_photoUrl) { //save image source $this->_image->addReference($this->_photoUrl); $this->printDebug("Saved image reference"); } } //create party membership relationships if ($partyName) { //if membership relationship with party doesn't exist, create it $partyRel = LsQuery::getByModelAndFieldsQuery('Relationship', array('entity1_id' => $member->id, 'entity2_id' => $partyEntity->id, 'category_id' => RelationshipTable::MEMBERSHIP_CATEGORY))->fetchOne(); if (!$partyRel) { $partyRel = new Relationship(); $partyRel->Entity1 = $member; $partyRel->Entity2 = $partyEntity; $partyRel->setCategory('Membership'); $modified = $partyRel->getAllModifiedFields(); $partyRel->save(); $partyRel->addReference($this->_profileUrlBase . $member->bioguide_id, null, $modified, 'Congressional Biographical Directory'); $this->printDebug("Created membership in political party: " . $partyName); } } $senator = null; //create senate relationships foreach ($this->_senateRelationships as $rel) { $modified = $rel->getAllModifiedFields(); $rel->save(); $rel->addReference($this->_profileUrlBase . $member->bioguide_id, null, $modified, 'Congressional Biographical Directory'); $this->printDebug("Saved Senate relationship"); } //create house relationships foreach ($this->_houseRelationships as $rel) { $modified = $rel->getAllModifiedFields(); $rel->save(); $rel->addReference($this->_profileUrlBase . $member->bioguide_id, null, $modified, 'Congressional Biographical Directory'); $this->printDebug("Saved House relationship"); } //save everything if (!$this->testMode) { $this->db->commit(); } } catch (Exception $e) { $this->db->rollback(); throw $e; } }
protected function addRelationship($org, $category_id, $description) { try { $category = Doctrine::getTable('RelationshipCategory')->find($category_id); if (!$category) { $this->setError('No relationships added -- you must choose a category'); return false; } $this->db->beginTransaction(); foreach ($this->edits as &$edit) { $person_id = $edit['New Person'] ? $edit['New Person'] : $edit['Existing Person']; $person = Doctrine::getTable('Entity')->find($person_id); $q = LsDoctrineQuery::create()->from('Relationship r')->where('r.entity1_id = ? and r.entity2_id = ?', array($person->id, $org->id))->fetchOne(); if ($q) { unset($edit['Relationship']); continue; } $rel = new Relationship(); $rel->Entity1 = $person; $rel->Entity2 = $org; $rel->setCategory($category); $rel->is_current = 1; if (isset($edit['Relationship']['start_date']) && preg_match('/\\d\\d\\d\\d/', $edit['Relationship']['start_date'])) { $rel->start_date = $edit['Relationship']['start_date']; } if (isset($edit['Relationship']['end_date']) && preg_match('/\\d\\d\\d\\d/', $edit['Relationship']['end_date'])) { $rel->end_date = $edit['Relationship']['end_date']; } if ($category->name == 'Position' && isset($edit['Relationship']['title'])) { $rel->description1 = $edit['Relationship']['title']; } else { if ($description) { $description == trim($description); $rel->description1 = $description; if ($description == 'Director' || $description == 'Trustee') { $rel->is_board = 1; $rel->is_employee = 0; } } } if (isset($edit['Relationship']['notes'])) { $rel->notes = $edit['Relationship']['notes']; } $rel->save(); $rel->addReference($this->source_url, null, null, $this->source_name); $this->printDebug($rel . ' saved'); unset($edit['Relationship']); } $this->db->commit(); } catch (Exception $e) { $this->db->rollback(); throw $e; } }
private function importFiling($org, $lda_filing) { try { $this->printTimeSince(); $this->printDebug('Starting import...'); $excerpt = array(); //$time = microtime(1); $this->db->beginTransaction(); $date = null; $excerpt['Federal Filing Id'] = $lda_filing->federal_filing_id; $excerpt['Year'] = $lda_filing->year; $excerpt['Type'] = $lda_filing->LdaType->description; if (preg_match('/^[^T]*/su', $lda_filing->received, $match)) { $date = $match[0]; $date = str_replace('/', '-', $date); } $lda_registrant = Doctrine::getTable('LdaRegistrant')->find($lda_filing->registrant_id); $excerpt['Registrant'] = $lda_registrant->name; if ($lda_filing->client_id) { $lda_client = Doctrine::getTable('LdaClient')->find($lda_filing->client_id); $excerpt['Client'] = $lda_client->name; } else { $this->db->rollback(); return null; } $lobbying_entity = null; //DETERMINE (& CREATE) LOBBYING ENTITY //$this->printTimeSince(); //$this->printDebug('determine/create...'); if (strtolower(OrgTable::stripNamePunctuation($lda_client->name)) == strtolower(OrgTable::stripNamePunctuation($lda_registrant->name))) { $lobbying_entity = $org; $client_entity = null; if (!$lobbying_entity->lda_registrant_id) { $lobbying_entity->lda_registrant_id = $lda_registrant->federal_registrant_id; $lobbying_entity->save(); $lobbying_entity->addReference(self::$filing_url . $lda_filing->federal_filing_id, null, $lobbying_entity->getAllModifiedFields(), 'LDA Filing', null, $date, false); } else { if ($lobbying_entity->lda_registrant_id != $lda_registrant->federal_registrant_id) { $this->printDebug("LDA registrant ids did not match up for {$lobbying_entity->name} and {$lda_registrant->name} even though names matched {$lda_client->name}\n"); $this->db->rollback(); return null; } } $this->printDebug($lobbying_entity->name . ' noted (same as client ' . $lda_client->name . ')'); } else { $client_entity = $org; if ($lda_client->description) { $description = trim($lda_client->description); if ($description != '' && preg_match('/[\\/\\-]\\d+[\\/\\-]/isu', $description) == 0) { if (strlen($description) < 200) { if (!$org->blurb || $org->blurb == '') { $org->blurb = $description; } } else { if (!$org->summary || $org->summary == '') { $org->summary = $description; } } } } $org->save(); $this->printDebug($lda_client->name . ' is distinct from ' . $lda_registrant->name); } $lda_lobbyists = $lda_filing->LdaLobbyists; $excerpt['Lobbyists'] = array(); foreach ($lda_lobbyists as $lda_lobbyist) { $excerpt['Lobbyists'][] = $lda_lobbyist->name; } $excerpt['Lobbyists'] = implode('; ', $excerpt['Lobbyists']); if (!$lobbying_entity) { $lobbyist_name = null; if (count($lda_lobbyists)) { $lobbyist_parts = explode(',', $lda_lobbyists[0]->name); if (count($lobbyist_parts) > 1) { $lobbyist_last = trim($lobbyist_parts[0]); $arr = LsString::split($lobbyist_parts[1]); $lens = array_map('strlen', $arr); arsort($lens); $keys = array_keys($lens); $lobbyist_longest = $arr[$keys[0]]; $lobbyist_name = trim($lobbyist_parts[1]) . ' ' . trim($lobbyist_parts[0]); $existing_lobbyist_registrant = null; } else { $lobbyist_name = preg_replace('/^(Mr|MR|MS|Dr|DR|MRS|Mrs|Ms)\\b\\.?/su', '', $lda_lobbyists[0]->name); $arr = LsString::split(trim($lobbyist_name)); $arr = LsArray::strlenSort($arr); $lobbyist_last = array_pop($arr); if (count($arr)) { $lobbyist_longest = array_shift(LsArray::strlenSort($arr)); } else { $lobbyist_longest = ''; } } } //check to see if registrant and lobbyist are same if (count($lda_lobbyists) == 1 && (strtoupper($lda_lobbyists[0]->name) == strtoupper($lda_registrant->name) || $lobbyist_last && stripos($lda_registrant->name, $lobbyist_last) == strlen($lda_registrant->name) - strlen($lobbyist_last) && stristr($lda_registrant->name, $lobbyist_longest))) { $existing_lobbyist_registrant = EntityTable::getByExtensionQuery('Lobbyist')->addWhere('lobbyist.lda_registrant_id = ?', $lda_registrant->federal_registrant_id)->execute()->getFirst(); if ($existing_lobbyist_registrant) { $lobbying_entity = $existing_lobbyist_registrant; $this->printDebug('Existing lobbyist is lobbying entity: ' . $lobbying_entity->name); } else { $lobbyist = $this->prepLobbyistName($lda_lobbyists[0]->name); if ($lobbyist) { $lobbyist->lda_registrant_id = $lda_registrant->federal_registrant_id; $lobbyist->save(); $lobbyist->addReference(self::$filing_url . $lda_filing->federal_filing_id, null, $lobbyist->getAllModifiedFields(), 'LDA Filing', null, $date, false); $this->printDebug('New lobbyist/lobbying entity saved: ' . $lobbyist->name); $lobbying_entity = $lobbyist; } } } else { if ($existing_firm = EntityTable::getByExtensionQuery('Org')->addWhere('org.lda_registrant_id = ? ', $lda_registrant->federal_registrant_id)->execute()->getFirst()) { $modified = array(); $lobbying_entity = $existing_firm; if ($lda_registrant->description) { $description = trim($lda_registrant->description); if ($description != '' && preg_match('/[\\/\\-]\\d+[\\/\\-]/isu', $description) == 0) { if (strlen($description) < 200) { if (!$existing_firm->blurb || $existing_firm->blurb == '') { $existing_firm->blurb = $description; $modified[] = 'blurb'; } } else { if (!$existing_firm->summary || $existing_firm->summary == '') { $existing_firm->summary = $description; $modified[] = 'summary'; } } } } if ($lda_registrant->address && $lda_registrant->address != '' && count($existing_firm->Address) == 0) { if ($address = $existing_firm->addAddress($lda_registrant->address)) { $existing_firm->save(); $address->addReference(self::$filing_url . $lda_filing->federal_filing_id, null, $address->getAllModifiedFields(), 'LDA Filing', null, $date, false); } } $existing_firm->save(); if (count($modified)) { $existing_firm->addReference(self::$filing_url . $lda_filing->federal_filing_id, null, $modified, 'LDA Filing', null, $date, false); } $this->printDebug('Existing firm is lobbying entity: ' . $lobbying_entity->name); } else { $firm = new Entity(); $firm->addExtension('Org'); $firm->addExtension('Business'); $firm->addExtension('LobbyingFirm'); $firm->name = LsLanguage::titleize(OrgTable::stripNamePunctuation($lda_registrant->name), true); $firm->lda_registrant_id = $lda_registrant->federal_registrant_id; if ($lda_registrant->description) { $description = trim($lda_registrant->description); if ($description != '' && preg_match('/[\\/\\-]\\d+[\\/\\-]/isu', $description) == 0) { if (strlen($description) < 200) { $firm->blurb = $description; } else { $firm->summary = $description; } } } if ($lda_registrant->address && $lda_registrant->address != '') { if ($address = $firm->addAddress($lda_registrant->address)) { $firm->save(); $address->addReference(self::$filing_url . $lda_filing->federal_filing_id, null, $address->getAllModifiedFields(), 'LDA Filing', null, $date, false); } } $firm->save(); $this->printDebug('New lobbying firm/lobbying entity saved: ' . $firm->name); $firm->addReference(self::$filing_url . $lda_filing->federal_filing_id, null, $firm->getAllModifiedFields(), 'LDA Filing', null, $date, false); $lobbying_entity = $firm; } } } //PREP GOVT ENTITIES //$this->printTimeSince(); //$this->printDebug('gov entities...'); $lda_govts = $lda_filing->LdaGovts; //$this->printDebug('count of lda govs is ***** ' . count($lda_govts)); $govt_entities = array(); $excerpt['Government Bodies'] = array(); foreach ($lda_govts as $lda_govt) { $excerpt['Government Bodies'][] = $lda_govt->name; $name_arr = $this->prepGovtName($lda_govt->name); if (!$name_arr) { continue; } if ($govt_entity = EntityTable::findByAlias($lda_govt->name, $context = 'lda_government_body')) { $govt_entities[] = $govt_entity; //$this->printDebug('Existing govt entity: ' . $govt_entity->name); } else { if ($govt_entity = EntityTable::getByExtensionQuery(array('Org', 'GovernmentBody'))->addWhere('name = ?', array($name_arr[0]))->fetchOne()) { $govt_entities[] = $govt_entity; $alias = new Alias(); $alias->context = 'lda_government_body'; $alias->name = $lda_govt->name; $alias->entity_id = $govt_entity->id; $alias->save(); } else { $govt_entity = new Entity(); $govt_entity->addExtension('Org'); $govt_entity->addExtension('GovernmentBody'); $govt_entity->name = $name_arr[0]; $govt_entity->name_nick = $name_arr[1]; $govt_entity->is_federal = 1; $govt_entity->save(); $alias = new Alias(); $alias->context = 'lda_government_body'; $alias->name = $lda_govt->name; $alias->entity_id = $govt_entity->id; $alias->save(); $govt_entity->addReference(self::$filing_url . $lda_filing->federal_filing_id, $excerpt, $govt_entity->getAllModifiedFields(), 'LDA Filing', null, $date, false); $govt_entities[] = $govt_entity; } } } $excerpt['Government Bodies'] = implode('; ', $excerpt['Government Bodies']); $excerpt_str = ''; foreach ($excerpt as $k => $v) { $excerpt_str .= $k . ": "; $excerpt_str .= $v . "\n"; } $excerpt = trim($excerpt_str); $this->printDebug($excerpt); $relationships = array(); $lobbying_entity_extensions = $lobbying_entity->getExtensions(); //CREATE LOBBYIST POSITION RELATIONSHIPS //$this->printTimeSince(); //$this->printDebug('lobbyist positions...'); $category = Doctrine::getTable('RelationshipCategory')->findOneByName('Position'); if (!in_array('Lobbyist', $lobbying_entity_extensions)) { $firm_lobbyists = array(); if ($lobbying_entity->exists()) { $q = LsDoctrineQuery::create()->from('Entity e')->leftJoin('e.Relationship r ON (r.entity1_id = e.id)')->where('r.entity2_id = ? AND r.category_id = ?', array($lobbying_entity->id, RelationshipTable::POSITION_CATEGORY)); $firm_lobbyists = $q->execute(); } $lobbyists = array(); foreach ($lda_lobbyists as $lda_lobbyist) { $lobbyist = $this->prepLobbyistName($lda_lobbyist->name); if (!$lobbyist) { continue; } $existing_lobbyist = null; foreach ($firm_lobbyists as $fl) { if (PersonTable::areNameCompatible($fl, $lobbyist)) { $existing_lobbyist = $fl; break; } } //echo "before lobb save or rel save: "; //$this->printTimeSince(); if (!$existing_lobbyist) { $lobbyist->save(); $lobbyist->addReference(self::$filing_url . $lda_filing->federal_filing_id, $excerpt, $lobbyist->getAllModifiedFields(), 'LDA Filing', null, $date, false); //$this->printDebug('New lobbyist saved: ' . $lobbyist->name); $r = new Relationship(); $r->Entity1 = $lobbyist; $r->Entity2 = $lobbying_entity; $r->setCategory('Position'); $r->description1 = 'Lobbyist'; $r->is_employee = 1; $r->save(); $r->addReference(self::$filing_url . $lda_filing->federal_filing_id, $excerpt, $lobbyist->getAllModifiedFields(), 'LDA Filing', null, $date, false); //$this->printDebug('New position relationship saved: ' . $lobbying_entity->name . ' and ' . $lobbyist->name); $lobbyists[] = $lobbyist; } else { //$this->printDebug('Lobbyist exists: ' . $lobbyist->name . ' is same as ' . $existing_lobbyist->name); $lobbyists[] = $existing_lobbyist; } } } //PREP ISSUES //$this->printTimeSince(); //$this->printDebug('issues...'); $issues = array(); $lda_issues = Doctrine_Query::create()->from('LdaFilingIssue f')->leftJoin('f.LdaIssue i')->where('f.filing_id = ?', $lda_filing->id)->execute(); foreach ($lda_issues as $lda_issue) { $name = LsLanguage::nameize($lda_issue->LdaIssue->name); if (!($issue = Doctrine::getTable('LobbyIssue')->findOneByName($name))) { $issue = new LobbyIssue(); $issue->name = $name; $issue->save(); //$this->printDebug('Lobbying issue saved: ' . $issue->name); } $issues[] = array($issue, $lda_issue->specific_issue); } //CREATE LOBBY FILING //$this->printTimeSince(); //$this->printDebug('creating lobby filing:'); $lobby_filing = new LobbyFiling(); $lobby_filing->year = $lda_filing->year; $lobby_filing->amount = $lda_filing->amount; $lobby_filing->federal_filing_id = $lda_filing->federal_filing_id; $period = $lda_filing->LdaPeriod->description; $lobby_filing->start_date = $date; if ($paren = strpos($period, '(')) { $lobby_filing->period = trim(substr($period, 0, $paren)); } else { $lobby_filing->period = 'Undetermined'; } $lobby_filing->report_type = LsLanguage::nameize($lda_filing->LdaType->description); foreach ($issues as $issue) { $filing_issue = new LobbyFilingLobbyIssue(); $filing_issue->Issue = $issue[0]; $filing_issue->Filing = $lobby_filing; $filing_issue->specific_issue = $issue[1]; $filing_issue->save(); } if (in_array('Lobbyist', $lobbying_entity_extensions)) { $lobby_filing->Lobbyist[] = $lobbying_entity; //$this->printDebug('Lobbying entity lobbyist added to lobbying relationship: ' . $lobbying_entity->name); } else { foreach ($lobbyists as $lobbyist) { $lobby_filing->Lobbyist[] = $lobbyist; } } //var_dump($lobby_filing->toArray()); $lobby_filing->save(); //CREATE TRANSACTION RELATIONSHIP, IF ANY //$this->printTimeSince(); //$this->printDebug('starting transaction relationships:'); $transaction = null; if ($client_entity != null) { $transaction = RelationshipTable::getByCategoryQuery('Transaction')->addWhere('r.entity1_id = ?', $client_entity->id)->addWhere('r.entity2_id = ?', $lobbying_entity->id)->addWhere('transaction.is_lobbying = ?', 1)->fetchOne(); if ($transaction) { $transaction->updateDateRange($date, true); if ($lda_filing->amount && $lda_filing->amount != '') { if (!$transaction->amount || $transaction->amount == '') { $transaction->amount = $lda_filing->amount; } else { $transaction->amount += $lda_filing->amount; } } $transaction->filings++; $transaction->save(); $transaction->addReference(self::$filing_url . $lda_filing->federal_filing_id, $excerpt, $transaction->getAllModifiedFields(), 'LDA Filing', null, $date, false); } else { $transaction = new Relationship(); $transaction->Entity1 = $client_entity; $transaction->Entity2 = $lobbying_entity; $transaction->setCategory('Transaction'); $transaction->description1 = 'Lobbying Client'; $transaction->is_lobbying = 1; $transaction->filings = 1; $transaction->updateDateRange($date, true); if (in_array('Person', $lobbying_entity_extensions)) { $transaction->description2 = 'Hired Lobbyist'; } else { $transaction->description2 = 'Lobbying Firm'; } if ($lda_filing->amount && $lda_filing->amount != '') { $transaction->amount = $lda_filing->amount; } $transaction->save(); $transaction->addReference(self::$filing_url . $lda_filing->federal_filing_id, $excerpt, $transaction->getAllModifiedFields(), 'LDA Filing', null, $date, false); //$this->printDebug('New lobbying transaction saved between client ' . $client_entity->name . ' and lobbying firm ' . $lobbying_entity->name); } $relationships[] = $transaction; } //CREATE LOBBYING RELATIONSHIP //$this->printTimeSince(); //$this->printDebug('starting lobbying relationships:'); foreach ($govt_entities as $govt_entity) { $lobbying_relationship = RelationshipTable::getByCategoryQuery('Lobbying')->addWhere('r.entity1_id = ?', $lobbying_entity->id)->addWhere('r.entity2_id = ?', $govt_entity->id)->fetchOne(); if ($lobbying_relationship) { $lobbying_relationship->updateDateRange($date); $lobbying_relationship->filings++; $lobbying_relationship->save(); } else { $lobbying_relationship = new Relationship(); $lobbying_relationship->Entity1 = $lobbying_entity; $lobbying_relationship->Entity2 = $govt_entity; $lobbying_relationship->setCategory('Lobbying'); if ($transaction) { $lobbying_relationship->description1 = 'Lobbying (for client)'; } else { $lobbying_relationship->description1 = 'Direct Lobbying'; } $lobbying_relationship->description2 = $lobbying_relationship->description1; $lobbying_relationship->updateDateRange($date, true); $lobbying_relationship->filings = 1; $lobbying_relationship->save(); $lobbying_relationship->addReference(self::$filing_url . $lda_filing->federal_filing_id, $excerpt, $lobbying_relationship->getAllModifiedFields(), 'LDA Filing', null, $date, false); } $relationships[] = $lobbying_relationship; } foreach ($relationships as $relationship) { $lobby_filing->Relationship[] = $relationship; } $lobby_filing->save(); //$this->printTimeSince(); $this->printDebug("Import Completed\n"); $this->db->commit(); } catch (Exception $e) { $this->db->rollback(); throw $e; } }
public function parseResults($match) { if (isset($match['bio'])) { $bio_dirty = LsHtml::replaceEntities(LsString::spacesToSpace(LsHtml::stripTags($match['bio'], "; "))); $bio_dirty = preg_replace('/(\\;\\s)+/is', '; ', $bio_dirty); } foreach ($match as $k => &$m) { $m = LsHtml::replaceEntities(LsString::spacesToSpace(LsHtml::stripTags($m, " "))); } if (isset($match['name'])) { $name = $match['name']; $bio = ''; if (isset($match['bio'])) { $bio = $match['bio']; } } else { return; } $this->printDebug("_________________________\n\nname: " . $name . "\n"); $this->printDebug("bio: " . $bio . "\n"); $accept = strtolower($this->readline('Process this entity? (n to skip) ')); if ($accept == 'n' || $accept == 'no') { return false; } if (!$this->org_org) { if ($this->last_first) { $entity = PersonTable::parseCommaName($name); } else { $entity = PersonTable::parseFlatName($name); } $similar_entities = PersonTable::getSimilarQuery2($entity)->execute(); } else { $entity = new Entity(); $entity->addExtension('Org'); foreach ($this->org_extensions as $ext) { $entity->addExtension($ext); } $entity->setEntityField('name', $name); $name = trim($name); $name = str_replace('.', '', $name); $similar_entities = OrgTable::getSimilarQuery($entity)->execute(); } $matched = false; foreach ($similar_entities as $similar_entity) { if ($similar_entity['primary_ext'] == 'Person') { $this->printDebug(' POSSIBLE MATCH: ' . $similar_entity->name . ' (Orgs :: ' . $similar_entity->getRelatedOrgsSummary() . " Bio :: {$similar_entity->summary})"); } else { $this->printDebug(' POSSIBLE MATCH: ' . $similar_entity->name . ' (Summary :: ' . $similar_entity->summary . ')'); } $accept = $this->readline(' Is this the same entity? (y or n)'); $attempts = 1; while ($accept != 'y' && $accept != 'n' && $attempts < 5) { $accept = $this->readline(' Is this the same entity? (y or n) '); $attempts++; } if ($accept == 'y') { $entity = $similar_entity; $matched = true; $this->printDebug(' [accepted]'); //sleep(1); break; } else { if ($accept == 'break') { break; } } } $created = false; if (!$matched) { if ($entity->getPrimaryExtension() == 'Person') { $this->printDebug(' New person: ' . $entity->name_first . ' ' . $entity->name_last); } else { $this->printDebug(' New org: ' . $entity->name); } $accept = $this->readline(' create this new entity? (y or n) '); $attempts = 1; while ($accept != 'y' && $accept != 'n' && $attempts < 5) { $accept = $this->readline(' create this new entity? (y or n) '); $attempts++; } if ($accept == 'y') { if ($entity->getPrimaryExtension() == 'Person') { $this->printDebug("\n Bio: {$bio} \n"); $accept = $this->readline(' Add this bio? (y or n) '); $attempts = 1; while ($accept != 'y' && $accept != 'n' && $attempts < 5) { $accept = $this->readline(' add this bio? (y or n) '); $attempts++; } if ($accept == 'y') { $entity->summary = $bio; } } $entity->save(); $entity->addReference($this->url, null, null, $this->url_name); $created = true; $this->printDebug(' ' . $entity->name . ' saved'); //sleep(1); } } if (($matched || $created) && $entity->getPrimaryExtension() == 'Person') { $accept = $this->readline("Parse above bio for possible relationships? (y or n) "); $attempts = 1; while ($accept != 'y' && $accept != 'n' && $attempts < 5) { $accept = $this->readline("Parse above bio for possible relationships? (y or n) "); $attempts++; } if ($accept == 'y') { $names = $entity->parseBio($bio_dirty); $this->printDebug(" Orgs that {$entity} has a position at?"); foreach ($names as $name) { $exists = false; $name = trim($name); $accept = $this->readline(" > {$name} :: an org? (y or n or b to break) "); $attempts = 1; $accept = strtolower($accept); while ($accept != 'y' && $accept != 'n' && $accept != 'b' && $attempts < 5) { $accept = $this->readline(" {$name} :: an org? (y or n or b to break) "); $accept = strtolower($accept); $attempts++; } if ($accept == 'b') { break; } else { if ($accept == 'y') { $this->printDebug(' .....looking for names.....'); $orgs = EntityTable::getByExtensionAndNameQuery('Org', $name)->limit(10)->execute(); $related_org = null; foreach ($orgs as $org) { $q = LsDoctrineQuery::create()->from('Relationship r')->where('entity1_id = ? and entity2_id = ?', array($entity->id, $org->id))->fetchOne(); if ($q) { $this->printDebug(' Position already exists, skipping...'); $exists = true; break; } $accept = $this->readline(" Create a position relationship between {$entity->name} and {$org->name}? (y or n) "); $attempts = 1; while ($accept != 'y' && $accept != 'n' && $attempts < 5) { $accept = $this->readline(" Create a position relationship between {$entity->name} and {$org->name}? (y or n) "); $attempts++; } if ($accept == 'y') { $related_org = $org; break; } } if (!$related_org && !$exists) { $accept = $this->readline(" couldn't find org, should this one be created: {$name} (y or n) "); while ($accept != 'y' && $accept != 'n' && $attempts < 5) { $accept = $this->readline(" couldn't find org, should this one be created: {$name} (y or n) "); $attempts++; } if ($accept == 'y') { $related_org = new Entity(); $related_org->addExtension('Org'); $related_org->name = preg_replace('/\\.(?!com)/i', '', $name); $extensions = $this->readline(" what extensions should this org get? (eg 'Business, LobbyingFirm, LawFirm') "); $extensions = preg_split('/\\,\\s*/isu', $extensions, -1, PREG_SPLIT_NO_EMPTY); try { foreach ($extensions as $extension) { $related_org->addExtension($extension); } $related_org->save(); $related_org->addReference($this->url, null, null, $this->url_name); } catch (Exception $e) { $this->printDebug(' !!! problems with org creation, skipping'); $related_org = null; } } } if ($related_org) { $q = LsDoctrineQuery::create()->from('Relationship r')->where('r.entity1_id = ? and r.entity2_id = ? and r.category_id = ?', array($entity->id, $related_org->id, 1))->fetchOne(); if ($q) { $this->printDebug(' (relationship already found, skipping...)'); continue; } $relationship = new Relationship(); $relationship->Entity1 = $entity; $relationship->Entity2 = $related_org; $relationship->setCategory('Position'); $title = $this->readline(" Title for this position relationship? (<enter> to skip) "); if (strlen($title) > 2) { $relationship->description1 = $title; } $current = strtolower($this->readline(" Is the relationship current? (y or n or <enter> to skip) ")); if (in_array($current, array('y', 'yes'))) { $relationship->is_current = 1; } else { if (in_array($current, array('n', 'no'))) { $relationship->is_current = 0; } } $board = strtolower($this->readline(" Is the relationship a board position? (y or n or <enter> to skip) ")); if (in_array($board, array('y', 'yes'))) { $relationship->is_board = 1; } else { if (in_array($board, array('n', 'no'))) { $relationship->is_board = 0; } } $relationship->save(); $relationship->addReference($this->url, null, null, $this->url_name); $this->printDebug(" Relationship saved: {$relationship}"); } } } } } } if ($matched || $created) { if ($this->list) { $q = LsDoctrineQuery::create()->from('LsListEntity l')->where('l.entity_id = ? and l.list_id = ?', array($entity->id, $this->list->id))->fetchOne(); if (!$q) { $le = new LsListEntity(); $le->Entity = $entity; $le->LsList = $this->list; if (isset($match['rank'])) { if (preg_match('/(\\d+)/isu', $match['rank'], $m)) { $le->rank = $m[1]; } } $le->save(); $this->printDebug('List membership saved'); } } if ($this->org) { $q = LsDoctrineQuery::create()->from('Relationship r')->where('r.entity1_id = ? and r.entity2_id = ? and r.category_id = ?', array($entity->id, $this->org->id, 1))->fetchOne(); if ($q) { $this->printDebug(' (relationship already found, skipping...)'); return; } $relationship = new Relationship(); $relationship->Entity1 = $entity; $relationship->Entity2 = $this->org; $relationship->setCategory($this->relationship_category); if ($this->description1) { $relationship->description1 = $this->description1; } else { $description = $this->readline(" what description to give this relationship ({$relationship}) ? (less than 3 chars will skip)"); if (strlen($description) > 2) { $relationship->description1 = $description; } } if ($this->relationship_category == 'Position') { $relationship->is_board = $this->is_board; } else { if ($this->relationship_category == 'Donation') { if ($this->amount) { $relationship->amount = $this->amount; } else { $amount = $this->readline(" what amount ({$relationship}) ? (less than 3 chars will skip)"); if (strlen($amount) > 1) { $relationship->amount = $amount; } } } } $relationship->save(); $relationship->addReference($this->url, null, null, $this->url_name); $this->printDebug(" Relationship saved: {$relationship}"); } } //dump history if (isset($match['affiliation1'])) { $affiliation = $match['affiliation']; //$this->printDebug($affiliation); } }
private function importRelationship($person, $position = null, $current = null, $is_board, $person_arr) { $q = LsDoctrineQuery::create()->from('Relationship r')->leftJoin('r.Position p')->where('r.entity1_id = ?', $person->id)->addWhere('r.entity2_id = ?', $this->entity->id)->addWhere('r.category_id = ?', RelationshipTable::POSITION_CATEGORY)->orderBy('r.id ASC'); if ($is_board) { $q->addWhere('p.is_board = 1'); } else { $q->addWhere('p.is_executive = 1'); //$q->addWhere('r.description1 = ?', $position) // ->addWhere('p.id IS NOT NULL'); } $count = $q->count(); if ($count == 0) { //if relationship doesn't already exist, create it $rel = new Relationship(); $rel->entity1_id = $person->id; $rel->entity2_id = $this->entity->id; $rel->setCategory('Position'); $rel->is_current = $current; if ($is_board) { $rel->is_board = 1; $rel->is_executive = $position != 'Chairman' && $position != 'Director' ? 1 : 0; $rel->description1 = null; //better not to set description if is_board = 1 } else { $rel->description1 = $position; $rel->is_executive = 1; $rel->is_board = 0; } if (!$this->testMode) { $rel->save(); //save sources $rel->addReference($person_arr['readableXmlUrl'], null, null, $this->entity->name . ' ' . $person_arr['formName'], null, $person_arr['date']); if (isset($this->filing_url) && isset($this->filing_name)) { $rel->addReference($this->filing_url, null, null, $this->entity->name . " " . $this->filing_name, null, $this->filing_date); } } $this->printDebug("+ Relationship created: " . $rel->id . " (" . ($rel->is_board ? "Board Member" : $rel->description1) . ")"); } elseif ($count == 1) { //this part might be redundant because importRoster() updates expired board memberships at the end? if ($is_board) { //if one relationship exists, expire if necessary $rel = $q->fetchOne(); if ($current == false && $rel->is_current) { $rel->is_current = false; if (!$this->testMode) { $rel->save(); } $this->printDebug("Existing relationship no longer current: " . $rel->id . " (" . $rel->Entity1->name . ")"); } } } else { //if mmultiple existing relationships found, do nothing /* $rel = $q->fetchOne(); if ($is_board) { //for board relationship... //if existing rel was last updated by a bot, we update it if (($current != $rel->is_current) && ($rel->last_user_id < 4)) { $rel->is_current = $current; $rel->addReference($this->filing_url, null, $fields= array('is_current'), $this->entity->name . " " . $this->filing_name, null, $this->filing_date); $rel->save(); $this->printDebug("Board relationship updated: " . $rel); } } else if ($current && !$rel->is_current && ($rel->last_user_id < 4)) { $rel->is_current = $current; $rel->addReference($this->filing_url, null, $fields= array('is_current'), $this->entity->name . " " . $this->filing_name, null, $this->filing_date); $rel->save(); $this->printDebug("Exec relationship updated: " . $rel); } */ } }
public function processRow($row) { if (isset($row['url']) && $row['url'] != '' && isset($row['url_name']) && $row['url_name'] != '') { $url = $row['url']; $url_name = $row['url_name']; } else { $url = $this->url; $url_name = $this->url_name; } foreach ($row as &$r) { trim($r); } unset($r); if ($this->entity) { $required = array('entity_name', 'primary_type', 'relationship_category'); } else { $required = array('entity_name', 'primary_type'); } foreach ($required as $req) { if (!isset($row[$req]) || $row[$req] == '') { $this->printDebug('!!! > skipping row, ' . $req . ' not set'); return; } } if ($row['primary_type'] != 'Person' && $row['primary_type'] != 'Org') { $this->printDebug('!!! > primary type not properly set, skipping row...'); return; } if ($this->entity) { $relationship_category = trim($row['relationship_category']); $relationship_category_id = array_search($relationship_category, RelationshipCategoryTable::$categoryNames); if (!$relationship_category_id) { $this->printDebug('!!! > relationship type not properly set, skipping row...'); return; } } $this->printDebug("processing: " . $row['entity_name'] . '......'); if ($row['primary_type'] == 'Person') { $entity2 = PersonTable::parseFlatName($row['entity_name']); $similar_entities = PersonTable::getSimilarQuery2($entity2)->execute(); } else { $entity2 = new Entity(); $entity2->addExtension('Org'); $entity2->setEntityField('name', $row['entity_name']); $similar_entities = OrgTable::getOrgsWithSimilarNames($entity2->name); } $matched = false; foreach ($similar_entities as $similar_entity) { if ($similar_entity['primary_ext'] == 'Person') { $this->printDebug(' POSSIBLE MATCH: ' . $similar_entity->name . ' (Orgs :: ' . $similar_entity->getRelatedOrgsSummary() . " Bio :: {$similar_entity->summary})"); } else { $this->printDebug(' POSSIBLE MATCH: ' . $similar_entity->name . ' (Summary :: ' . $similar_entity->summary . ')'); } $accept = $this->readline(' Is this the same entity? (y or n or b to break)'); if ($accept == 'y') { $entity2 = $similar_entity; $matched = true; $this->printDebug(' [accepted]'); break; } else { if ($accept == 'b') { break; } } } $created = false; if (!$matched) { if ($entity2->getPrimaryExtension() == 'Person') { $this->printDebug(' New person: ' . $entity2->name_first . ' ' . $entity2->name_last); } else { $this->printDebug(' New org: ' . $entity2->name); } $accept = $this->readline(' create this new entity? (y or n) '); if ($accept == 'y') { try { $extensions = LsString::split($row['entity_extensions'], '\\s*\\,\\s*'); foreach ($extensions as $extension) { $entity2->addExtension($extension); } $entity2->save(); $entity2->addReference($url, null, null, $url_name); } catch (Exception $e) { $this->printDebug(' !!! problems with extensions for this row'); } $fields = array('summary', 'blurb', 'website'); foreach ($fields as $field) { if (isset($row[$field])) { $entity2[$field] = $row[$field]; } } $entity2->save(); $entity2->addReference($url, null, null, $url_name); $created = true; $this->printDebug(' ' . $entity2->name . ' saved'); //sleep(1); } else { $entity2 = null; } } // create relationship if ($entity2) { if ($this->entity) { $relationship = new Relationship(); if (isset($row['relationship_order']) && $row['relationship_order'] != '') { if ($row['relationship_order'] == '1') { $relationship->Entity1 = $this->entity; $relationship->Entity2 = $entity2; } else { $relationship->Entity2 = $this->entity; $relationship->Entity1 = $entity2; } } else { if ($relationship_category == 'Position' || $relationship_category == 'Education') { if ($row['primary_type'] == 'Org') { $relationship->Entity1 = $this->entity; $relationship->Entity2 = $entity2; } else { $relationship->Entity1 = $entity2; $relationship->Entity2 = $this->entity; } } else { $relationship->Entity1 = $this->entity; $relationship->Entity2 = $entity2; } } $relationship->setCategory($relationship_category); $cols = array('description1', 'description2', 'start_date', 'end_date', 'goods', 'amount', 'is_board', 'is_executive', 'is_employee'); foreach ($cols as $col) { if (isset($row[$col]) && $row[$col] != '') { try { $relationship[$col] = $row[$col]; } catch (Exception $e) { $this->printDebug(" could not set {$col} for relationship, skipping"); } } } $q = LsDoctrineQuery::create()->from('Relationship r')->where('r.entity1_id = ? and r.entity2_id = ? and r.category_id = ? and r.id <> ?', array($relationship->entity1_id, $relationship->entity2_id, $relationship->category_id, $relationship->id))->fetchOne(); if ($q) { $this->printDebug(' (relationship already found, skipping...)'); return; } $relationship->save(); $relationship->addReference($url, null, null, $url_name); $this->printDebug(" Relationship saved: {$relationship}\n"); } else { if ($this->list) { $q = LsDoctrineQuery::create()->from('LsListEntity le')->where('le.entity_id = ? and le.list_id = ?', array($entity2->id, $this->list->id))->fetchOne(); if ($q) { $this->printDebug(' (already on list, skipping...)'); return; } $le = new LsListEntity(); $le->LsList = $this->list; $le->Entity = $entity2; var_dump($row); if (isset($row['rank'])) { echo $row['rank']; $le->rank = $row['rank']; } $le->save(); } } } }
private function importRelationship($person, $corp, $position, $person_arr) { $r = new Relationship(); $r->entity1_id = $person->id; $r->entity2_id = $corp->id; $r->setCategory('Position'); $r->is_current = 1; $r->description1 = $position; $q = LsDoctrineQuery::create()->from('Relationship r')->where('r.entity1_id = ?', $person->id)->addWhere('r.entity2_id = ?', $corp->id)->addWhere('r.category_id = ?', RelationshipTable::POSITION_CATEGORY)->addWhere('r.description1 = ?', $r->description1); if ($q->count() == 0) { //$r->relationship_id = $r->id; if ($person_arr['isDirector'] == '1') { $r->is_board = 1; $r->is_executive = $position != 'Chairman' && $position != 'Director' ? 1 : 0; } else { $r->is_executive = 1; $r->is_board = 0; } $r->save(); $r->addReference($person_arr['form4Url'], null, $fields = array('entity1_id', 'entity2_id', 'is_current', 'description1', 'description2', 'category_id', 'relationship_id', 'is_board', 'is_executive'), $corp->name . ' Form 4', null, $person_arr['date']); $r->addReference($person_arr['proxyUrl'], null, $fields = array('entity1_id', 'entity2_id', 'is_current', 'description1', 'description2', 'category_id', 'relationship_id', 'is_board', 'is_executive'), $corp->name . ' proxy, ' . $person_arr['proxyYear']); } else { $this->printDebug('relationship exists'); } }
private function getTransactionDetails($org, $fedspending_id) { if (!$this->browser->get($this->fedSpendingSearchUrl, array('record_id' => $fedspending_id, 'datype' => 'X', 'detail' => '4'))->responseIsError()) { $response_url = $this->browser->getUrlInfo(); $response_url_str = "http://" . $response_url['host'] . $response_url['path'] . "?" . $response_url['query']; $this->printDebug("Transaction #{$fedspending_id} http://" . $response_url['host'] . $response_url['path'] . "?" . $response_url['query']); $text = $this->browser->getResponseText(); $this->browser->setResponseText(iconv('ISO-8859-1', 'UTF-8', $this->browser->getResponseText())); $xml = $this->browser->getResponseXml(); } else { $this->printDebug("Couldn't get " . $this->fedSpendingSearchUrl); return false; } $obligated_amount = $xml->xpath('/fedspendingSearchResults/data/record/amounts/obligatedAmount'); $maj_agency_cat = $xml->xpath('/fedspendingSearchResults/data/record/purchaser_information/maj_agency_cat'); $contracting_agency = $xml->xpath('/fedspendingSearchResults/data/record/purchaser_information/contractingOfficeAgencyID'); $psc_cat = $xml->xpath('/fedspendingSearchResults/data/record/product_or_service_information/psc_cat'); $signed_date = $xml->xpath('/fedspendingSearchResults/data/record/contract_information/signedDate'); $descriptionOfContractRequirement = $xml->xpath('/fedspendingSearchResults/data/record/contract_information/descriptionOfContractRequirement'); $current_completion_date = $xml->xpath('/fedspendingSearchResults/data/record/contract_information/currentCompletionDate'); $state_code = $xml->xpath('/fedspendingSearchResults/data/record/principal_place_of_performance/stateCode'); $place_of_performance_congressional_district = $xml->xpath('/fedspendingSearchResults/data/record/principal_place_of_performance/placeOfPerformanceCongressionalDistrict'); foreach ($obligated_amount as $key => $dont_use) { $gov_name = $this->cleanGovtBodyName($maj_agency_cat[$key]); $gov_agency = $this->getGovernmentBodyEntity($gov_name, $maj_agency_cat[$key]); if ($gov_agency) { //$this->printDebug("Found existing Government Agency" . $gov_agency->name); } else { $gov_name = $this->cleanGovtBodyName($maj_agency_cat[$key]); $this->printDebug($gov_name); $gov_agency = $this->addGovernmentBodyEntity($gov_name, $maj_agency_cat[$key]); $this->printDebug("Creating new Government Agency: " . $gov_agency->name); $gov_agency->addReference(self::$baseContractorUrl . $org->fedspending_id, null, array('name', 'parent_id'), 'FedSpending.org'); } if (!$gov_agency) { $this->printDebug("Error creating Government Agency"); return false; } $sub_agency_name = $this->cleanGovtBodyName($contracting_agency[$key]); $sub_agency = $this->getGovernmentBodyEntity($sub_agency_name, $contracting_agency[$key]); if ($sub_agency_name == $gov_agency->name) { $sub_agency = $gov_agency; } if (!$sub_agency) { $sub_agency = $this->addGovernmentBodyEntity($sub_agency_name, $contracting_agency[$key], $gov_agency->id); $this->printDebug("Creating new sub-agency: " . $sub_agency->name); $sub_agency->addReference(self::$baseContractorUrl . $org->fedspending_id, null, array('name', 'parent_id'), 'FedSpending.org'); } if (!$sub_agency) { $this->printDebug("Error creating Government Agency"); return false; } try { $district = null; $state = explode(': ', $state_code[$key]); $federal_district = explode(': ', $place_of_performance_congressional_district[$key]); $state = $state[0]; $federal_district = $federal_district[0]; $filing = new FedspendingFiling(); $filing->goods = $descriptionOfContractRequirement[$key]; $filing->amount = abs($obligated_amount[$key]); if ($filing->amount < 1000) { $this->printDebug('amount under $1000, rolling back'); return 'under_1000'; } //$this->printDebug("state: " . $state . " and district: " . $federal_district); if ($district = PoliticalDistrictTable::getFederalDistrict($state, $federal_district)) { //$this->printDebug("found " . $district->id); $filing->District = $district; } elseif (trim($state) && trim($federal_district)) { try { $district = PoliticalDistrictTable::addFederalDistrict($state, $federal_district); $this->printDebug("Adding District " . $state . " #" . $district->id); $filing->District = $district; } catch (Exception $e) { throw $e; } } $filing->fedspending_id = $fedspending_id; $filing->start_date = LsDate::formatFromText($signed_date[$key]); $filing->end_date = LsDate::formatFromText($current_completion_date[$key]); $relationship = null; if ($relationship = $org->getRelationshipsWithQuery($sub_agency, RelationshipTable::TRANSACTION_CATEGORY)->fetchOne()) { $relationship->addFedspendingFiling($filing); } else { $relationship = new Relationship(); $relationship->Entity1 = $org; $relationship->Entity2 = $sub_agency; $relationship->setCategory('Transaction'); $relationship->description1 = 'Contractor'; $relationship->description2 = 'Client'; $relationship->save(); $relationship->addReference(self::$baseContractorUrl . $org->fedspending_id, null, array('start_date', 'end_date', 'amount', 'goods'), 'FedSpending.org'); $relationship->addFedspendingFiling($filing); } $filing->save(); return true; } catch (Exception $e) { throw $e; } } }