public function getURLLastModified($url, $symlink, $bucket)
{
$this->clearstatcache();
if (!$this->fileExists($symlink)) {
return array('time' => 0, 'revalidate' => true, 'cache_age' => 0);
}
$symlinkAbs = $this->absolutize($symlink);
$stat = lstat($symlinkAbs);
$now = time();
if (readlink($symlinkAbs) == '/dev/null') {
$failed_time = $now - $stat['mtime'];
if ($failed_time > binarypool_config::getBadUrlExpiry()) {
unlink($symlinkAbs);
return array('time' => 0, 'revalidate' => true, 'cache_age' => $failed_time);
}
$failed_nextfetch = $stat['mtime'] + binarypool_config::getBadUrlExpiry() - $now;
throw new binarypool_exception(122, 400, "File download failed {$failed_time} seconds ago. Re-fetching allowed in next time in {$failed_nextfetch} seconds: {$url}");
}
$cache_age = $now - $stat['mtime'];
$revalidate = false;
if ($cache_age > binarypool_config::getCacheRevalidate($bucket)) {
$revalidate = true;
}
return array('time' => filemtime($symlinkAbs), 'revalidate' => $revalidate, 'cache_age' => $cache_age);
}
public function getURLLastModified($url, $symlink, $bucket)
{
$symlink .= '.link';
$now = $this->time;
if (!$this->fileExists($symlink)) {
return array('time' => 0, 'revalidate' => true, 'cache_age' => 0);
}
$contents = $this->getFile($symlink);
$contents = json_decode($contents, true);
if ($contents['link'] == '/dev/null') {
// Dead URL
$failed_time = $now - $contents['mtime'];
if ($failed_time > binarypool_config::getBadUrlExpiry()) {
$this->unlink($symlink);
return array('time' => 0, 'revalidate' => true, 'cache_age' => $failed_time);
}
$failed_nextfetch = $contents['mtime'] + binarypool_config::getBadUrlExpiry() - $now;
throw new binarypool_exception(122, 400, "File download failed {$failed_time} seconds ago. Re-fetching allowed in next time in {$failed_nextfetch} seconds: {$url}");
}
$cache_age = $now - $contents['mtime'];
$revalidate = false;
if ($cache_age > binarypool_config::getCacheRevalidate($bucket)) {
$revalidate = true;
}
return array('time' => $contents['mtime'], 'revalidate' => $revalidate, 'cache_age' => $cache_age);
}
function testUpdatedSymlink()
{
$this->assignLastModified();
$cache_age = binarypool_config::getCacheRevalidate('test') + 1;
binarypool_views::$lastModified->setReturnValue('lastModified', array('cache_age' => $cache_age));
$asset = $this->createMockAsset();
$storage = $this->createMockStorage($asset);
$storage->expectCallCount('symlink', 2);
$storage->expectCallCount('relink', 1);
$this->assignMockStorageFactory($storage);
binarypool_views::created('test', 'foo', array('URL' => 'http://local.ch/foo.gif'));
}
protected function getFilesFromUrl()
{
$url = $this->request->getParam('URL');
$this->log->debug("Downloading file: %s", $url);
if (!self::$lastModified) {
self::$lastModified = new binarypool_lastmodified();
}
$lastmodified = self::$lastModified->lastModified($this->bucket, $url);
if (binarypool_config::getCacheRevalidate($this->bucket) === 0) {
$lastmodified['time'] = 0;
}
$tmpfile = tempnam(sys_get_temp_dir(), 'binary');
if ($tmpfile == '' || $tmpfile === FALSE) {
throw new binarypool_exception(104, 500, "Could not create temporary file");
}
array_push($this->tmpfiles, $tmpfile);
$result = array('code' => 0, 'headers' => array(), 'body' => '');
$retries = 3;
if ($lastmodified['revalidate']) {
$httpc = new binarypool_httpclient();
while ($retries) {
try {
$result = $httpc->download($url, $tmpfile, $lastmodified['time']);
if ($result['code'] < 500) {
break;
}
} catch (binarypool_httpclient_exception $e) {
// ignore - dropped connections etc. - retry
$this->log->debug("Failed download attempt from %s: %s", $url, $e);
}
sleep(1);
$retries--;
}
} else {
$result['code'] = 304;
}
if (304 == $result['code']) {
$this->log->debug("File %s has not been modified", $url);
return array();
}
if ($result['code'] != 200 || !filesize($tmpfile)) {
binarypool_views::flagBadUrl($this->bucket, $url);
throw new binarypool_exception(121, 400, "File could not be fetched from URL: " . $url);
}
$url_parsed = parse_url($url);
$filename = basename($url_parsed['path']);
# Restrict filenames TO ALPHANUMS and reduce sequences of '.' to avoid
# traversal issues, unicode issues, command injection etc.
$filename = preg_replace(array('#\\.{2,}#', '#[^a-zA-Z0-9\\.]+#'), array('.', '_'), $filename);
return array('_' => array('file' => $tmpfile, 'filename' => $filename));
}
/**
* Create the URL view symlink for files which were downloaded
*
* @param $asset: An object of binarypool_asset.
*/
private static function linkURL($bucket, $asset, $metadata)
{
if (empty($metadata['URL'])) {
return;
}
$assetDir = '../../' . self::getCleanedBasepath($asset);
$symlink = self::getDownloadedViewPath($bucket, $metadata['URL']);
if (!self::$lastModified) {
self::$lastModified = new binarypool_lastmodified();
}
$lastmodified = self::$lastModified->lastModified($bucket, $metadata['URL']);
if ($lastmodified['cache_age'] > binarypool_config::getCacheRevalidate($bucket)) {
self::refreshLink($bucket, $assetDir, $symlink);
} else {
self::createLink($bucket, $assetDir, $symlink);
}
}
