/** * Will trigger API requests for the specified Website $idSite, * for the specified $period, for all segments that are pre-processed for this website. * Requests are triggered using cURL multi handle * * @param $idSite int * @param $period * @param $lastTimestampWebsiteProcessed * @return bool True on success, false if some request failed */ private function archiveVisitsAndSegments($idSite, $period, $lastTimestampWebsiteProcessed) { $timer = new Timer(); $url = $this->piwikUrl; $date = $this->getApiDateParameter($idSite, $period, $lastTimestampWebsiteProcessed); $url .= $this->getVisitsRequestUrl($idSite, $period, $date); $url .= self::APPEND_TO_API_REQUEST; $visitsInLastPeriods = $visitsLastPeriod = 0; $success = true; $urls = array(); $noSegmentUrl = $url; // already processed above for "day" if ($period != "day") { $urls[] = $url; $this->requests++; } foreach ($this->getSegmentsForSite($idSite) as $segment) { $urlWithSegment = $url . '&segment=' . urlencode($segment); $urls[] = $urlWithSegment; $this->requests++; } $cliMulti = new CliMulti(); $cliMulti->setAcceptInvalidSSLCertificate($this->acceptInvalidSSLCertificate); $cliMulti->setConcurrentProcessesLimit($this->getConcurrentRequestsPerWebsite()); $response = $cliMulti->request($urls); foreach ($urls as $index => $url) { $content = array_key_exists($index, $response) ? $response[$index] : null; $success = $success && $this->checkResponse($content, $url); if ($noSegmentUrl === $url && $success) { $stats = @unserialize($content); if (!is_array($stats)) { $this->logError("Error unserializing the following response from {$url}: " . $content); } $visitsInLastPeriods = $this->getVisitsFromApiResponse($stats); $visitsLastPeriod = $this->getVisitsLastPeriodFromApiResponse($stats); } } // we have already logged the daily archive above if ($period != "day") { $this->logArchivedWebsite($idSite, $period, $date, $visitsInLastPeriods, $visitsLastPeriod, $timer); } return $success; }
public function test_request_shouldRequestAllUrls_IfMultipleUrlsAreGiven_WithHighConcurrentRequestLimit() { $urls = $this->buildUrls('getPiwikVersion', 'getAnswerToLife', 'getPiwikVersion'); $this->cliMulti->setConcurrentProcessesLimit(10); $this->assertRequestReturnsValidResponses($urls, array('getPiwikVersion', 'getAnswerToLife', 'getPiwikVersion')); }
/** * Will trigger API requests for the specified Website $idSite, * for the specified $period, for all segments that are pre-processed for this website. * Requests are triggered using cURL multi handle * * @param $idSite int * @param $period string * @param $date string * @param $archiveSegments bool Whether to pre-process all custom segments * @return bool True on success, false if some request failed */ private function archiveReportsFor($idSite, $period, $date, $archiveSegments) { $timer = new Timer(); $url = $this->getVisitsRequestUrl($idSite, $period, $date, $segment = false); $url = $this->makeRequestUrl($url); $visitsInLastPeriods = $visitsLastPeriod = 0; $success = true; $urls = array(); $noSegmentUrl = $url; // already processed above for "day" if ($period != "day") { $urls[] = $url; $this->logArchiveWebsite($idSite, $period); } $segmentRequestsCount = 0; if ($archiveSegments) { $urlsWithSegment = $this->getUrlsWithSegment($idSite, $period, $date); $urls = array_merge($urls, $urlsWithSegment); $segmentRequestsCount = count($urlsWithSegment); // in case several segment URLs for period=range had the date= rewritten to the same value, we only call API once $urls = array_unique($urls); } $this->requests += count($urls); $cliMulti = new CliMulti(); $cliMulti->setAcceptInvalidSSLCertificate($this->acceptInvalidSSLCertificate); $cliMulti->setConcurrentProcessesLimit($this->getConcurrentRequestsPerWebsite()); $response = $cliMulti->request($urls); foreach ($urls as $index => $url) { $content = array_key_exists($index, $response) ? $response[$index] : null; $success = $success && $this->checkResponse($content, $url); if ($noSegmentUrl === $url && $success) { $stats = @unserialize($content); if (!is_array($stats)) { $this->logError("Error unserializing the following response from {$url}: " . $content); } if ($period == 'range') { // range returns one dataset (the sum of data between the two dates), // whereas other periods return lastN which is N datasets in an array. Here we make our period=range dataset look like others: $stats = array($stats); } $visitsInLastPeriods = $this->getVisitsFromApiResponse($stats); $visitsLastPeriod = $this->getVisitsLastPeriodFromApiResponse($stats); } } $this->logArchivedWebsite($idSite, $period, $date, $segmentRequestsCount, $visitsInLastPeriods, $visitsLastPeriod, $timer); return $success; }