private function setList()
{
$name = 'Fortune 1000 (' . $this->year . ')';
$list = Doctrine_Query::create()->from('LsList L')->where('L.name = ?', $name)->fetchOne();
//if thlis year's fortune list doesn't already exist, create it
if (!$list) {
try {
$this->db->beginTransaction();
$list = new LsList();
$list->name = $name;
$list->description = "Fortune Magazine's list of the 1000 US companies with the largest published revenue figures.";
$list->is_ranked = 1;
$list->save();
$this->list = $list;
$ref = new Reference();
$ref->object_model = 'LsList';
$ref->object_id = $list->id;
$ref->fields = 'name, description, is_ranked';
$ref->source = 'http://money.cnn.com/magazines/fortune/fortune500/' . $this->year . '/full_list/';
$ref->name = 'Fortune Magazine Online';
$ref->save();
if (!$this->testMode) {
$this->db->commit();
}
} catch (Exception $e) {
$this->db->rollback();
throw $e;
}
} else {
$this->list = $list;
}
}
public function executeAdd($request)
{
$this->list_form = new LsListForm();
$this->reference_form = new ReferenceForm();
if ($request->isMethod('post')) {
$params = $request->getParameter('list');
$refParams = $request->getParameter('reference');
$this->list_form->bind($params);
$this->reference_form->bind($refParams);
if ($this->list_form->isValid() && $this->reference_form->isValid()) {
$list = new LsList();
$list->name = $params['name'];
$list->description = $params['description'];
$list->is_ranked = isset($params['is_ranked']) ? true : false;
if ($this->getUser()->hasCredential('admin')) {
$list->is_admin = isset($params['is_admin']) && $params['is_admin'] ? true : false;
$list->is_featured = isset($params['is_featured']) && $params['is_featured'] ? true : false;
}
$list->saveWithRequiredReference($refParams);
$this->redirect($list->getInternalUrl());
}
}
}
public function execute()
{
if (!$this->safeToRun('uk-mp-candidates')) {
$this->printDebug('Script already running');
die;
}
// Get (or create) the UK local Network
$uk = Doctrine::getTable('LsList')->findOneByName('United Kingdom');
if (!$uk) {
$uk = new LsList();
$uk->name = 'United Kingdom';
$uk->is_network = 1;
$uk->description = 'People and organizations with significant influence on the policies of the United Kingdom';
$uk->display_name = 'uk';
$uk->save();
}
// Get the MP list
$raw = $this->getMPs();
// Add new MPs to the list
foreach ($raw as $mp) {
$this->printDebug(sprintf('Processing %s', $mp['name']));
// Split name
$entity = PersonTable::parseFlatName($mp['name']);
$entity->blurb = 'Prospective Parliamentary Candidate for ' . $mp['constituency'];
$q = TagTable::getByTripleQuery('yournextmp', 'url', $mp['url']);
$r = $q->count();
if ($r) {
$this->printDebug('Already processed, skipping.');
continue;
}
// Get political party
$q = EntityTable::getByExtensionQuery('PoliticalParty')->addWhere('e.name = ?', $mp['party']);
if (!($partyEntity = $q->fetchOne())) {
$partyEntity = new Entity();
$partyEntity->addExtension('Org');
$partyEntity->addExtension('PoliticalParty');
$partyEntity->name = $mp['party'];
$partyEntity->blurb = 'UK Political Party';
$partyEntity->save(null, true, array($uk->id));
$this->printDebug("Created new political party: " . $mp['party']);
}
// Save entity to UK Network
$entity->party_id = $partyEntity->id;
$entity->save(null, true, array($uk->id));
// Add party relationship
$r = new Relationship();
$r->entity1_id = $entity->id;
$r->entity2_id = $partyEntity->id;
$r->setCategory('Membership');
$r->description1 = 'Prospective parliamentary candidate';
$r->is_current = true;
// $r->start_date = // Don't know where we can get this, and "now" seems kind of wrong
$r->save();
// Add YourNextMP triple
$entity->addTagByTriple('yournextmp', 'url', $mp['url']);
// Add references
$ref = new Reference();
$ref->addFields(array('name_first', 'name_last', 'name_middle'));
// Don't need this
$ref->source = $mp['url'];
$ref->name = 'YourNextMP.com - ' . $entity['name'];
$ref->object_model = 'Entity';
$ref->object_id = $entity->getId();
$ref->save();
unset($ref);
$ref = new Reference();
$ref->addFields(array('name'));
$ref->source = $mp['party_url'];
$ref->name = 'YourNextMP.com - ' . $partyEntity['name'];
$ref->object_model = 'Entity';
$ref->object_id = $partyEntity->getId();
$ref->save();
unset($ref);
$ref = new Reference();
$ref->addFields(array('name'));
$ref->source = $mp['url'];
$ref->name = 'YourNextMP.com - ' . $entity['name'];
$ref->object_model = 'Relationship';
$ref->object_id = $r->getId();
$ref->save();
unset($ref);
$r->free(true);
unset($r);
// Add image?
if ($mp['image']) {
if ($fileName = ImageTable::createFiles($mp['image'])) {
//insert image record
$image = new Image();
$image->filename = $fileName;
$image->title = $entity['name'];
$image->caption = 'From YourNextMP under CC-BY-SA license.';
$image->is_featured = true;
$image->is_free = true;
$image->url = $mp['image'];
$this->printDebug("Imported image: " . $image->filename);
}
$image->Entity = $entity;
$image->save();
if ($mp['image']) {
//save image source
$image->addReference($mp['image']);
$this->printDebug("Saved image reference");
}
unset($image);
}
// Add party image?
if ($mp['party_image']) {
if ($fileName = ImageTable::createFiles($mp['party_image'])) {
//insert image record
$partyImage = new Image();
$partyImage->filename = $fileName;
$partyImage->title = $partyEntity['name'];
$partyImage->caption = 'From YourNextMP under CC-BY-SA license.';
$partyImage->is_featured = true;
$partyImage->is_free = true;
$partyImage->url = $mp['party_image'];
$this->printDebug("Imported image: " . $partyImage->filename);
}
$partyImage->Entity = $partyEntity;
$partyImage->save();
if ($mp['party_image']) {
//save image source
$partyImage->addReference($mp['party_image']);
$this->printDebug("Saved image reference");
}
unset($partyImage);
}
unset($entity);
unset($partyEntity);
}
}
public function execute()
{
require sfConfig::get('sf_lib_dir') . '/vendor/votesmart/VoteSmart.php';
$this->_sunlightUrlBase = 'http://congress.api.sunlightfoundation.com/legislators?apikey=' . sfConfig::get('app_sunlight_api_key') . '&all_legislators=true&bioguide_id=';
if (!$this->safeToRun('congress')) {
$this->printDebug('script already running');
die;
}
//establish house id
if (!$this->_houseEntityId) {
if (!($house = Doctrine::getTable('Entity')->findOneByName($this->_houseEntityName))) {
$house = new Entity();
$house->addExtension('Org');
$house->addExtension('GovernmentBody');
$house->name = $this->_houseEntityName;
$house->save();
}
$this->_houseEntityId = $house->id;
}
//establish senate id
if (!$this->_senateEntityId) {
if (!($senate = Doctrine::getTable('Entity')->findOneByName($this->_senateEntityName))) {
$senate = new Entity();
$senate->addExtension('Org');
$senate->addExtension('GovernmentBody');
$senate->name = $this->_senateEntityName;
$senate->save();
}
$this->_senateEntityId = $senate->id;
}
//loop through sessions
while ($session = current($this->_sessions)) {
//make sure session hasn't already been scraped
if ($this->hasMeta($session, 'is_complete') && $this->getMeta($session, 'is_complete')) {
$this->printDebug("\n\nSession " . $session . " members already scraped; skipping");
next($this->_sessions);
continue;
}
if (!$this->_sessionListId) {
$sessionName = $session . 'th Congress';
if (!($sessionList = Doctrine::getTable('LsList')->findOneByName($sessionName))) {
$sessionList = new LsList();
$sessionList->name = $sessionName;
$sessionList->save();
}
$this->_sessionListId = $sessionList->id;
}
$this->printDebug("\n\nFetching bios for congress session: " . $session . "\n");
$this->loadExistingSessionMembers($session);
$this->printDebug("\n\ndone loading session members: " . $session . "\n");
$this->scrapeCongressSessionMembers($session);
if ($this->_completeSessionOnFinish) {
//end relationships for discontinuing members only after all current members have been processed
$this->updateDiscontinuingMemberRelationships($session);
$this->saveMeta($session, 'is_complete', true);
$this->removeMeta($session, 'last_processed');
$this->printDebug("\n\nSession " . $session . " completed");
}
$this->resetSessionVariables();
next($this->_sessions);
}
}
protected function setList($title, $description, $fields = 'name, description, is_ranked')
{
$count = count($this->urls);
$end = $this->end ? $this->end : $count;
reset($this->urls);
if (!$this->year) {
throw new Exception('ForbesScraper:setYear has to be called before ForbesScraper:setList');
}
$this->list_name = $title . " ({$this->year})";
//'Forbes Largest Private Companies (' . $this->year . ')';
$list = Doctrine_Query::create()->from('LsList L')->where('L.name = ?', $this->list_name)->fetchOne();
//if thlis year's fortune list doesn't already exist, create it
if (!$list) {
try {
$list = new LsList();
$list->name = $this->list_name;
$list->description = $description;
//"Fortune Magazine's list of large US private companies";
$list->is_ranked = 1;
$list->save();
$this->list = $list;
$ref = new Reference();
$ref->object_model = 'LsList';
$ref->object_id = $list->id;
$ref->fields = $fields;
$ref->source = $this->list_urls[$this->year]['source_url'];
$ref->save();
} catch (Exception $e) {
$this->db->rollback();
throw $e;
}
} else {
$this->list = $list;
}
}
