public function indexAction()
{
#Get request parameters
$election = $this->getRequest()->get('election') ?: self::DEFAULT_ELECTION;
$language = 'en';
$fresh = $this->getRequest()->get('fresh');
$format = $this->getRequest()->get('format') ?: 'html';
#Setup caching
$cacheKey = "{$election}.{$language}.{$format}";
$root = $this->get('kernel')->getRootDir();
require_once "{$root}/../src/Jlam/HorkosBundle/phpfastcache-final/phpfastcache.php";
phpFastCache::setup("storage", "auto");
$cache = new phpFastCache();
#If a cache is available, use it and return it right away.
$response = $cache->get($cacheKey);
if ($response !== null && $fresh !== 'yes') {
return $response;
}
/* Doing it with ACL:
if ($responseObject = $this->get('cache')->fetch($cacheKey)) {
$response = unserialize($responseObject);
return $response;
}
*/
# Get the engine class name
$engineClassName = self::getScrappingEngineClassName($election);
# Set the logger for the riding entity
Riding::setLogger($this->get('logger'));
#If not, slurp data
$engineClassName::initialize($this->container);
$engineClassName::scrape();
$engineClassName::validate();
### Prepare the rendering #####################################################
$summary = $engineClassName::getSummary();
$magnitude = 1;
if (!$engineClassName::getScraperError()) {
$partyTally = Riding::getPartyTally(FALSE)->getTally();
$jurisdictionTally = Riding::getJurisdictionTally()->getTally();
#$partyTallyWasted = $partyTally['wasted'];
#arsort($partyTallyWasted);
$summary['totalWastedVotes'] = $jurisdictionTally['wasted'];
$magnitude = Riding::calculateMagnitudeWinner();
} else {
$partyTally = array();
$jurisdictionTally = array();
$partyTallyWasted = array();
$summary['totalWastedVotes'] = null;
}
### Render ####################################################################
$response = $this->render("JlamHorkosBundle:Horkos:index.{$format}.twig", array('ridings' => Riding::getAllRdingsSorted(), 'partyTally' => $partyTally, 'jurisdiction' => $jurisdictionTally, 'summary' => $summary, 'election' => $election ? $election : self::DEFAULT_ELECTION, 'error' => $engineClassName::getScraperError(), 'magnitude' => $magnitude));
#Save into caching
$date = new \DateTime();
$date->modify('+' . self::CACHE_TTL_SECS . ' seconds');
$response->setPublic();
$response->setExpires($date);
$response->setMaxAge(self::CACHE_TTL_SECS);
$response->setSharedMaxAge(self::CACHE_TTL_SECS);
if ($format == 'csv') {
$response->headers->set('Content-Type', 'text/csv');
$response->headers->set('Content-Disposition', 'attachment; filename="export.csv"');
}
$cache->set($cacheKey, $response, self::CACHE_TTL_SECS);
#Return the controller
return $response;
}
public static function scrape($lang = 'en')
{
$logger = self::getLogger('Starting offline scrape...');
$langPathPart = self::langPathPartLookup($lang);
$ridingPaths = parent::getRidingPaths(self::JURISDICTION_SHORTHAND, $langPathPart);
$ridingCount = count($ridingPaths);
self::addLog("{$ridingCount} ridings found");
foreach ($ridingPaths as $i => $path) {
$sourceUrl = self::generateSource($lang, $i);
self::addLog("Getting results for riding {$i} {$path}...");
$riding = new Riding();
$riding->setSource($sourceUrl);
$html = file_get_contents($path);
$doc = new \DOMDocument();
self::setErrorHandler();
$doc->loadHTML($html);
self::setErrorHandler(TRUE);
$xpath = new \DOMXPath($doc);
$xPathQuery = '//*[@id="grdResultsucElectoralDistrictResult' . $i . '"]/caption';
self::addLog("xPath: {$xPathQuery}");
$ridingNode = $xpath->query($xPathQuery);
$ridingName = trim(substr($ridingNode->item(0)->textContent, 50));
self::addLog($ridingName);
$riding->setName(utf8_decode($ridingName));
$tables = $doc->getElementsByTagName('table');
// nodes = $xpath->query('/html/body/div[2]/div[2]/div[2]/div[3]/table/tbody');
// cho "For " . $ridingNames[$i] . "\n";
// ar_export($tables->item(0)->textContent);
// cho "\n\n";
$tablesLength = $tables->length;
self::addLog("Found {$tablesLength} items in \$tables");
$rows = $tables->item(0)->getElementsByTagName('tr');
$numRows = $rows->length;
self::addLog("There are {$numRows} rows\n");
$j = 0;
for ($j = 0; $j < $numRows - 1; $j++) {
if ($j == 0) {
continue;
}
$row = $rows->item($j);
$cells = $row->getElementsByTagName('td');
$party = $cells->item(0)->textContent;
if (strpos($party, ':') !== FALSE) {
continue;
}
$votes = preg_replace("/[^0-9]/", "", $cells->item(2)->textContent);
$votes = str_replace(",", "", $votes);
self::addLog("Scrapped: {$party}\t{$votes}\n");
$riding->setVotes($party, $votes);
}
$xPathQuery = '//*[@id="divElectorNumberucElectoralDistrictResult' . $i . '"]/p';
$numVoters = trim(substr($xpath->query($xPathQuery)->item(0)->textContent, 80));
$numVoters = str_replace(',', '', $numVoters);
self::addLog("Number of voters: {$numVoters}");
$riding->setEligibleVoters($numVoters);
$row = $rows->item($numRows - 1);
$cells = $row->getElementsByTagName('td');
$totalVotes = $cells->item(2)->textContent;
$totalVotes = str_replace(',', '', $totalVotes);
self::addLog("Number of total votes: {$totalVotes}");
$riding->setAllRidingVotes($totalVotes);
$riding->updateTallies();
}
}
public static function scrape()
{
$logger = self::getLogger('Starting scrape...');
$ridingIdentfiers = self::getRidingIdentifiers();
self::addLog('Got ' . count($ridingIdentfiers) . ' ridings: ' . join(', ', $ridingIdentfiers));
foreach ($ridingIdentfiers as $i) {
$url = self::getFinalPath($i);
self::addLog("Getting results for riding {$i} {$url}...");
$riding = new Riding();
$riding->setSource($url);
$ridingCount = count(Riding::getAllRidings());
self::addLog("Riding count is {$ridingCount}");
$html = @file_get_contents($url);
if ($html === FALSE) {
self::addLog("Warning: no content for riding {$i} at {$url}");
continue;
}
/*
$doc = new \DOMDocument ();
self::setErrorHandler();
$doc->loadHTML ( $html );
self::setErrorHandler(TRUE);
$xpath = new \DOMXPath ( $doc );
*/
$string = self::grep($html, 'Unofficial Poll Results', TRUE);
$ridingName = self::grep($string, 'Unofficial Poll Results - [0-9][0-9]* ([\\. A-Z-]*)')[0];
$ridingName = substr_replace($ridingName, '', -2);
self::addLog("Got ridingName: {$ridingName}");
$riding->setName($ridingName);
// nodes = $xpath->query('/html/body/div[2]/div[2]/div[2]/div[3]/table/tbody');
// cho "For " . $ridingNames[$i] . "\n";
// ar_export($tables->item(0)->textContent);
// cho "\n\n";
$string = self::grep($html, 'CHeadCA', TRUE);
preg_match_all("|<DIV CLASS=CHPA>([A-Z]*)</DIV>|", $string[0], $matches);
self::addLog('Got for party: ' . count($matches[1]) . ' matches: ' . join(',', $matches[1]));
$stringVotes = self::grep($html, 'ColFooter', TRUE);
preg_match_all("|<TD Class=ColFooter ALIGN=RIGHT VALIGN=TOP>([ 0-9,]*)<BR>|", $stringVotes[2], $matchesVotes);
self::addLog('Got for votes: ' . count($matchesVotes[1]) . ' matches: ' . join(' ', $matchesVotes[1]));
if (count($matches == 0)) {
self::addError("No matches found for party");
}
foreach ($matches[1] as $index => $party) {
$votes = str_replace(',', '', $matchesVotes[1][$index]);
if (empty($votes)) {
self::addError("No votes found for {$party} in {$ridingName}");
}
self::addLog("Setting {$votes} votes for {$party} in {$ridingName}");
$riding->setVotes($party, $votes);
}
# Find and set eligeable voters
$stringVotes = self::grep($html, 'ColFooter', TRUE);
preg_match_all("|<TD Class=ColFooter ALIGN=RIGHT VALIGN=TOP>([0-9,]*)|", $stringVotes[1], $matchesVotes);
self::addLog('Got for voters count ' . count($matchesVotes[1]) . ' matches: ' . join(' ', $matchesVotes[1]));
$numVoters = str_replace(',', '', $matchesVotes[1][0]);
self::addLog("Number of voters: {$numVoters}");
$riding->setEligibleVoters($numVoters);
# Find and save all votes (aka total votes)
$stringVotes = self::grep($html, 'ColFooter', TRUE);
preg_match_all("|<TD Class=ColFooter ALIGN=RIGHT VALIGN=TOP>([ 0-9,]*)</TABLE>|", $stringVotes[5], $matchesVotes);
self::addLog('Got ' . count($matchesVotes[1]) . ' matches: ' . join(' ', $matchesVotes[1]));
$totalVotes = str_replace(',', '', $matchesVotes[1][0]);
self::addLog("Number of total votes: {$totalVotes}");
$riding->setAllRidingVotes($totalVotes);
# Update talies
$riding->updateTallies();
}
}