/**
* Convert a picture to a different format and/or resize it.
*
* @param $from: File path to the original file.
* @param $to: File path where the new file should be saved.
* @param $mime: MIME type of the original image.
* @param $maxWidth: Maximum width of the new image. No resizing
* is done if this is empty. Can be used without
* specifying maxHeight.
* @param $maxHeight: Maximum height of the new image. Must be
* used together with $maxWidth.
*/
protected static function convert($from, $to, $mime, $maxWidth = '', $maxHeight = '')
{
$cmd = binarypool_config::getUtilityPath('convert');
# Make sure all output images are in RGB. This handles incoming CMYK
# images which some browsers can't display.
$cmd .= ' -colorspace RGB';
switch ($mime) {
case 'image/pdf':
case 'application/pdf':
$cmd .= ' -trim';
break;
case 'image/gif':
break;
default:
$cmd .= ' -flatten';
break;
}
if ($maxWidth != '') {
$scale = intval($maxWidth);
if ($maxHeight != '') {
$scale .= 'x' . intval($maxHeight);
# Don't enlarge if the size is already smaller than resized version
$scale .= '>';
}
$cmd .= ' -resize ' . escapeshellarg($scale) . '';
}
if ($mime == 'image/jpeg') {
# Sharpen image
$cmd .= ' -unsharp 0.5x1';
}
$cmd = $cmd . ' ' . escapeshellarg($from) . ' ' . escapeshellarg($to);
$log = new api_log();
$log->debug("Resizing image using command: {$cmd}");
shell_exec($cmd);
}
/**
* Prepare a curl handle for the http request
*
* @param String $url the URL to fetch
* @param int $modified (optional) seconds since Unix epoch of local copy
* @return $curl curl handle
*/
protected function prepareCurl($url, $lastmodified = 0)
{
$curl = curl_init();
// See http://curl.haxx.se/libcurl/c/curl_easy_setopt.html
curl_setopt($curl, CURLOPT_URL, $url);
curl_setopt($curl, CURLOPT_USERAGENT, binarypool_config::getUseragent());
curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
// return data as string
curl_setopt($curl, CURLOPT_FOLLOWLOCATION, 0);
// don't follow redirects
curl_setopt($curl, CURLOPT_CONNECTTIMEOUT, $this->connectionTimeout);
// can't connect? fail fast...
curl_setopt($curl, CURLOPT_TIMEOUT, $this->timeout);
// total timeout
curl_setopt($curl, CURLOPT_NOSIGNAL, true);
curl_setopt($curl, CURLOPT_DNS_USE_GLOBAL_CACHE, FALSE);
curl_setopt($curl, CURLOPT_FILETIME, TRUE);
curl_setopt($curl, CURLOPT_HEADER, TRUE);
curl_setopt($curl, CURLOPT_ENCODING, '');
// sends Accept-Encoding for all suported encodings
if ($lastmodified) {
curl_setopt($curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
curl_setopt($curl, CURLOPT_TIMEVALUE, $lastmodified);
}
return $curl;
}
protected function touch($bucket, $uri)
{
$storage = new binarypool_storage($bucket);
$assetFile = $uri;
if (!$storage->isFile($assetFile)) {
$assetFile .= '/index.xml';
if (!$storage->isFile($assetFile)) {
return false;
}
}
// Get TTL from request
$buckets = binarypool_config::getBuckets();
$ttl = $buckets[$bucket]['ttl'];
if ($this->request->getParam('TTL')) {
$newttl = intval($this->request->getParam('TTL'));
if ($newttl <= $ttl) {
// Don't allow higher TTL than bucket configuration
$ttl = $newttl;
}
}
// Set TTL
$oldAsset = $storage->getAssetObject($assetFile);
$asset = $storage->getAssetObject($assetFile);
$asset->setExpiry(time() + $ttl * 24 * 60 * 60);
$storage->saveAsset($asset, $assetFile);
// Update views
binarypool_views::updated($bucket, $assetFile, $oldAsset);
$this->setResponseCode(204);
return true;
}
function cleanSymlinks()
{
// Do symlink cleanup
printf("[%10s] Cleaning up symlinks.\n", 'FINAL');
$cmd = binarypool_config::getUtilityPath('symlinks');
system("{$cmd} -cdrs " . binarypool_config::getRoot() . "*/created");
system("{$cmd} -cdrs " . binarypool_config::getRoot() . "*/expiry");
system("{$cmd} -cdrsv " . binarypool_config::getRoot() . "*/downloaded |grep '/dev/null' |xargs -0 rm");
}
/**
* Convert a PDF/EPS to JPG/PNG.
*
* @param $from: File path to the original file.
* @param $to: File path where the new file should be saved.
* @param $format: Format to convert to.
*/
protected static function convert($from, $to, $format)
{
$cmd = binarypool_config::getUtilityPath('pdfconverter');
$cmd .= ' -f ' . $format;
$cmd .= ' ' . escapeshellarg($from) . ' ' . escapeshellarg($to);
$log = new api_log();
$log->debug("Rendering PDF with command: {$cmd}");
shell_exec($cmd);
}
protected function execute()
{
$xml = "<status method='get'>";
$buckets = binarypool_config::getBuckets();
foreach (array_keys($buckets) as $bucket) {
$xml .= '<bucket id="' . htmlspecialchars($bucket) . '" />';
}
$xml .= "</status>";
array_push($this->data, new api_model_xml($xml));
}
public function __construct($bucket)
{
$buckets = binarypool_config::getBuckets();
if (!isset($buckets[$bucket])) {
throw new binarypool_exception(100, 404, "Bucket not defined: {$bucket}");
}
$this->bucketName = $bucket;
$this->bucketConfig = $buckets[$bucket];
$this->storage = $this->getStorage($this->bucketConfig);
}
/**
* The getExpired call should return an asset which file expired
* yesterday.
*/
function testGetExpiredYesterday()
{
$assetId = '096dfa489bc3f21df56eded2143843f135ae967e';
$asset = $this->storage->save('IMAGE', array('_' => array('file' => $this->testfile)));
$date = date('Y/m/d', strtotime('-1 days'));
$viewToday = self::$BUCKET . 'expiry/' . $date . '/' . $assetId;
mkdir(dirname($viewToday), 0755, true);
symlink(dirname(binarypool_config::getRoot() . $asset), $viewToday);
$this->assertEqual(array('test/09/096dfa489bc3f21df56eded2143843f135ae967e/index.xml'), binarypool_browser::getExpired('test'));
}
protected static function convert($from, $to)
{
$cmd = binarypool_config::getUtilityPath('convert');
$cmd .= ' -density 288 -resize 220 +antialias';
# Make sure all output images are in RGB. This handles incoming CMYK
# images which some browsers can't display.
$cmd .= ' -colorspace RGB -trim ';
$cmd .= escapeshellarg($from) . ' ' . escapeshellarg($to);
$log = new api_log();
$log->debug("Converting PDF/EPS image using command: {$cmd}");
shell_exec($cmd);
}
function setUp()
{
parent::setUp();
// Remove bucket
$bucket = binarypool_config::getRoot() . 'test/';
if (file_exists($bucket)) {
$this->deltree($bucket);
}
// Remove trash
if (file_exists(binarypool_config::getRoot() . 'Trash/')) {
$this->deltree(binarypool_config::getRoot() . 'Trash/');
}
}
/**
* Validate original uploaded files and throw exceptions if they are
* invalid.
*
* @param $type: Type - the validation method is chosen based on this type.
* @param $bucket: Bucket and type define the validations that apply.
* @param $original: Absolute path to the original file.
* @return Boolean
*/
public static function validate($type, $bucket, $original)
{
$validationClass = "binarypool_validate_" . strtolower($type);
if (!class_exists($validationClass)) {
return true;
}
$config = null;
$buckets = binarypool_config::getBuckets();
if (isset($buckets[$bucket]['validations']) && isset($buckets[$bucket]['validations'][$type])) {
$config = $buckets[$bucket]['validations'][$type];
}
return call_user_func($validationClass . '::validate', $original, $config);
}
function setUp()
{
self::$BUCKET = binarypool_config::getRoot() . 'test/';
// Remove bucket
if (file_exists(self::$BUCKET)) {
$this->deltree(self::$BUCKET);
}
// Remove trash
if (file_exists(binarypool_config::getRoot() . 'Trash/')) {
$this->deltree(binarypool_config::getRoot() . 'Trash/');
}
$this->testfile = realpath(dirname(__FILE__) . '/res/vw_golf.jpg');
}
protected function execute()
{
$uri = $this->request->getPath();
// Deletions allowed on bucket?
$buckets = binarypool_config::getBuckets();
if (!isset($buckets[$this->bucket]['allowDeletions']) || $buckets[$this->bucket]['allowDeletions'] == false) {
throw new binarypool_exception(108, 403, "Deletions not allowed on this bucket.");
}
$storage = new binarypool_storage($this->bucket);
$storage->delete($uri);
$this->setResponseCode(204);
$this->response->send();
$this->ignoreView = true;
}
protected static function lint($source, $xsd = null)
{
$cmd = binarypool_config::getUtilityPath('xmllint');
$cmd .= ' --nowarning --nonet --noout';
if (!is_null($xsd)) {
$cmd .= ' --schema ' . escapeshellarg($xsd);
}
$cmd = $cmd . ' ' . escapeshellarg($source) . ' >/dev/null 2>/dev/null';
system($cmd, $retval);
if ($retval != 0) {
if (is_null($xsd)) {
throw new binarypool_exception(117, 400, "XML document is not well-formed.");
} else {
throw new binarypool_exception(118, 400, "XML document is not valid.");
}
}
return true;
}
function walk_dir($root, $callback, $exclude = array())
{
$fsroot = binarypool_config::getRoot();
$root = rtrim($root, '/') . '/';
$queue = array($root);
foreach ($exclude as &$path) {
$path = $root . trim($path, '/') . '/';
}
while ($base = array_shift($queue)) {
$relative = substr($base, strlen($fsroot));
$callback($relative);
if ($handle = opendir($base)) {
while (($child = readdir($handle)) !== FALSE) {
if (is_dir($base . $child) && $child != '.' && $child != '..') {
$combined_path = $base . $child . '/';
if (!in_array($combined_path, $exclude)) {
array_push($queue, $combined_path);
}
}
}
closedir($handle);
}
}
}
/**
* Create an asset XML with an absolute base path and reload it.
*/
function testAssetPersistanceWithAbsoluteBasePath()
{
$basepath = 'test/somehashhere/';
$absBasepath = binarypool_config::getRoot() . $basepath;
mkdir($absBasepath, 0755, true);
$asset = $this->testAssetWithAbsoluteBasePath();
file_put_contents($absBasepath . 'index.xml', $asset->getXML());
// Load XML from FS
$asset = new binarypool_asset($this->getDummyStorage(), $basepath . 'index.xml');
$xml = $asset->getXML();
$dom = DOMDocument::loadXML($xml);
$xp = new DOMXPath($dom);
// Test that the same information is still around
$this->assertXPath($xp, '/registry/@version', '3.0');
$this->assertXPath($xp, '/registry/items/item/location', 'http://bin.staticlocal.ch/vw_golf.jpg');
$this->assertXPath($xp, '/registry/items/item/location/@absolute', 'true');
$this->assertXPath($xp, '/registry/items/item/mimetype', 'image/jpeg');
$this->assertXPath($xp, '/registry/items/item/size', '51941');
return $asset;
}
/**
* Uses `file' on the command line to get the MIME type.
*/
protected static function getMimeTypeWithCmdLineFile($file)
{
$cmd = binarypool_config::getUtilityPath('file');
$mime = trim(shell_exec("{$cmd} -ib " . escapeshellarg($file)));
if ($mime == '' || $mime == 'regular file') {
return null;
} else {
return $mime;
}
}
/**
* Fix the extension of manually uploaded renditions.
*/
function testSaveRenditionFixExtension()
{
$this->assertFalse(file_exists(self::$BUCKET . 'cb/cbf9f9f453acaba556e00b48951815da5611f975/vw_golf_smaller.jpg'));
$asset = $this->storage->save('IMAGE', array('_' => array('file' => dirname(__FILE__) . '/../res/vw_golf_smaller.jpg'), 'detailpage' => array('file' => dirname(__FILE__) . '/../res/upload1.bin')));
$this->assertNotNull($asset);
$this->assertEqual('test/cb/cbf9f9f453acaba556e00b48951815da5611f975/index.xml', $asset);
$this->assertTrue(file_exists(self::$BUCKET . 'cb/cbf9f9f453acaba556e00b48951815da5611f975/vw_golf_smaller.jpg'), 'Original file was not written to file system.');
$this->assertTrue(file_exists(self::$BUCKET . 'cb/cbf9f9f453acaba556e00b48951815da5611f975/upload1.jpg'), 'detailpage rendition file was not written to file system.');
$this->assertFalse(file_exists(self::$BUCKET . 'cb/cbf9f9f453acaba556e00b48951815da5611f975/upload1.bin'), 'detailpage rendition file was written to file system with the wrong extension.');
$this->assertTrue(file_exists(binarypool_config::getRoot() . $asset), 'Asset file was not written to file system.');
}
protected function getFilesFromUrl()
{
$url = $this->request->getParam('URL');
$this->log->debug("Downloading file: %s", $url);
if (!self::$lastModified) {
self::$lastModified = new binarypool_lastmodified();
}
$lastmodified = self::$lastModified->lastModified($this->bucket, $url);
if (binarypool_config::getCacheRevalidate($this->bucket) === 0) {
$lastmodified['time'] = 0;
}
$tmpfile = tempnam(sys_get_temp_dir(), 'binary');
if ($tmpfile == '' || $tmpfile === FALSE) {
throw new binarypool_exception(104, 500, "Could not create temporary file");
}
array_push($this->tmpfiles, $tmpfile);
$result = array('code' => 0, 'headers' => array(), 'body' => '');
$retries = 3;
if ($lastmodified['revalidate']) {
$httpc = new binarypool_httpclient();
while ($retries) {
try {
$result = $httpc->download($url, $tmpfile, $lastmodified['time']);
if ($result['code'] < 500) {
break;
}
} catch (binarypool_httpclient_exception $e) {
// ignore - dropped connections etc. - retry
$this->log->debug("Failed download attempt from %s: %s", $url, $e);
}
sleep(1);
$retries--;
}
} else {
$result['code'] = 304;
}
if (304 == $result['code']) {
$this->log->debug("File %s has not been modified", $url);
return array();
}
if ($result['code'] != 200 || !filesize($tmpfile)) {
binarypool_views::flagBadUrl($this->bucket, $url);
throw new binarypool_exception(121, 400, "File could not be fetched from URL: " . $url);
}
$url_parsed = parse_url($url);
$filename = basename($url_parsed['path']);
# Restrict filenames TO ALPHANUMS and reduce sequences of '.' to avoid
# traversal issues, unicode issues, command injection etc.
$filename = preg_replace(array('#\\.{2,}#', '#[^a-zA-Z0-9\\.]+#'), array('.', '_'), $filename);
return array('_' => array('file' => $tmpfile, 'filename' => $filename));
}
/**
* Load settings from the config file.
*/
private static function load()
{
// Read correct config file based on environment
$env = '';
if (isset($_SERVER['BINARYPOOL_CONFIG'])) {
$env = $_SERVER['BINARYPOOL_CONFIG'];
}
$filename = API_PROJECT_DIR . "/conf/binarypool.php";
if ($env != '' && file_exists($filename)) {
$filename = API_PROJECT_DIR . '/conf/binarypool-' . $env . '.php';
}
include $filename;
// Get values
self::$buckets = $BUCKETS;
self::$root = $ROOT;
if (!file_exists(self::$root)) {
mkdir(self::$root);
if (!file_exists(self::$root)) {
throw new binarypool_exception(107, 500, "Binary Pool path is not available.");
}
}
self::$root = realpath(self::$root);
if (self::$root[strlen(self::$root) - 1] !== '/') {
self::$root .= '/';
}
self::$paths = $PATHS;
self::$badUrlExpiry = $BADURLEXPIRY;
self::$cacheRevalidate = $CACHEREVALIDATE;
self::$useragent = $USERAGENT;
}
protected function getS3Bucket()
{
$buckets = binarypool_config::getBuckets();
return $buckets['test_s3'];
}
/**
* Calculates the renditions for the given file.
*
* @param $type: Type - the renderer is chosen based on this type.
* @param $bucket: Bucket and type define the renditions which have
* to be generated.
* @param $original: Absolute path to the original file.
* @param $outputPath: Directory where the renditions are written to.
* @param $exclude: Renditions which don't have to be calculated.
* @param $assetFile: Path to the asset file. Needed for asynchronous
* rendition processes.
* @return Associative array of hashes. The key is the rendition name,
* the path is the relative path of the file.
*/
public static function render($type, $bucket, $original, $outputPath, $exclude = array(), $assetFile = null)
{
// Get configured renditions for this type
$buckets = binarypool_config::getBuckets();
$renditions = array();
if (!isset($buckets[$bucket]['renditions'][$type])) {
return $renditions;
}
// Information about the original file
$mime = binarypool_mime::getMimeType($original);
// Loops through the renditions configuration and generates the
// renditions as specified.
//
// The following keys of each rendition config are considered:
// - _sources: Ordered array of rendition names that should be
// used as input for this rendition. If not present
// of empty elements mean that the original
// binary is used.
// - _class: The render_base subclass to use. The class is
// prefixed with 'binarypool_render_' to get the
// definitive PHP class name.
// - _mimes: MIME type of the original binary for which this
// rendition is calculated.
//
// This three keys are removed from the array and the modified
// array is then passed to the render class as configuration.
//
// The list of renditions needs to be in the correct dependency
// order. If rendition `B' uses rendition `A' as a source, then
// rendition `A' must come before `B'.
foreach ($buckets[$bucket]['renditions'][$type] as $rendition => $renditionConfig) {
// Renditions that the user uploaded are not calculated
if (in_array($rendition, $exclude)) {
continue;
}
// Rendition to be calculated?
if (isset($renditionConfig['_mimes'])) {
$mimesCfg = $renditionConfig['_mimes'];
if (!is_array($mimesCfg)) {
$mimesCfg = array($mimesCfg);
}
if (!in_array($mime, $mimesCfg)) {
continue;
}
unset($renditionConfig['_mimes']);
}
// Get correct source file name
$sourceFile = null;
if (isset($renditionConfig['_sources'])) {
foreach ($renditionConfig['_sources'] as $sourceRendition) {
if (isset($renditions[$sourceRendition])) {
$sourceFile = $renditions[$sourceRendition];
break;
} else {
if ($sourceRendition === '') {
$sourceFile = $original;
break;
}
}
}
unset($renditionConfig['_sources']);
} else {
$sourceFile = $original;
}
if (is_null($sourceFile)) {
throw new binarypool_exception(106, 500, "Missing source rendition for rendition {$rendition}");
}
$renditionConfig['_bucket'] = $bucket;
// Get correct class name
$renditionType = strtolower($type);
if (isset($renditionConfig['_class'])) {
$renditionType = $renditionConfig['_class'];
unset($renditionConfig['_class']);
}
$renderingClass = "binarypool_render_" . $renditionType;
// Filenames
$renditionFile = $rendition;
$absoluteRendition = $outputPath . $renditionFile;
// Render image. Return value of rendering is the absolute path to the
// rendition including file extensions.
$absoluteRendition = call_user_func($renderingClass . '::render', $sourceFile, $absoluteRendition, $assetFile, $renditionConfig);
if (!is_null($absoluteRendition)) {
if (!file_exists($absoluteRendition)) {
throw new binarypool_exception(106, 500, "Could not create rendition: {$rendition}");
}
$renditions[$rendition] = $absoluteRendition;
}
}
return $renditions;
}
function testGetRenditionDirectory()
{
$storage = new binarypool_storage_driver_file();
$this->assertEqual($storage->getRenditionsDirectory('files/abc/def'), binarypool_config::getRoot() . 'files/abc/def/');
}
function testGetPathOfConvert()
{
$convert = binarypool_config::getUtilityPath('convert');
$this->assertNotEqual(0, strlen($convert));
}
function testUpdatedSymlink()
{
$this->assignLastModified();
$cache_age = binarypool_config::getCacheRevalidate('test') + 1;
binarypool_views::$lastModified->setReturnValue('lastModified', array('cache_age' => $cache_age));
$asset = $this->createMockAsset();
$storage = $this->createMockStorage($asset);
$storage->expectCallCount('symlink', 2);
$storage->expectCallCount('relink', 1);
$this->assignMockStorageFactory($storage);
binarypool_views::created('test', 'foo', array('URL' => 'http://local.ch/foo.gif'));
}
/**
* Tests that the correct rendition is saved.
*/
function testUploadRenditionVerify()
{
$this->testUploadRendition();
$this->get('/test/cb/cbf9f9f453acaba556e00b48951815da5611f975/index.xml');
$this->assertEqual(api_response::getInstance()->getCode(), 200);
$localHash = sha1_file(dirname(__FILE__) . '/../res/vw_golf_blur.jpg');
$remoteFile = $this->getText('/registry/items/item[rendition="detailpage"]/location');
$remoteHash = sha1_file(binarypool_config::getRoot() . '/' . $remoteFile);
$this->assertEqual($localHash, $remoteHash);
}
public function getURLLastModified($url, $symlink, $bucket)
{
$symlink .= '.link';
$now = $this->time;
if (!$this->fileExists($symlink)) {
return array('time' => 0, 'revalidate' => true, 'cache_age' => 0);
}
$contents = $this->getFile($symlink);
$contents = json_decode($contents, true);
if ($contents['link'] == '/dev/null') {
// Dead URL
$failed_time = $now - $contents['mtime'];
if ($failed_time > binarypool_config::getBadUrlExpiry()) {
$this->unlink($symlink);
return array('time' => 0, 'revalidate' => true, 'cache_age' => $failed_time);
}
$failed_nextfetch = $contents['mtime'] + binarypool_config::getBadUrlExpiry() - $now;
throw new binarypool_exception(122, 400, "File download failed {$failed_time} seconds ago. Re-fetching allowed in next time in {$failed_nextfetch} seconds: {$url}");
}
$cache_age = $now - $contents['mtime'];
$revalidate = false;
if ($cache_age > binarypool_config::getCacheRevalidate($bucket)) {
$revalidate = true;
}
return array('time' => $contents['mtime'], 'revalidate' => $revalidate, 'cache_age' => $cache_age);
}
public function getURLLastModified($url, $symlink, $bucket)
{
$this->clearstatcache();
if (!$this->fileExists($symlink)) {
return array('time' => 0, 'revalidate' => true, 'cache_age' => 0);
}
$symlinkAbs = $this->absolutize($symlink);
$stat = lstat($symlinkAbs);
$now = time();
if (readlink($symlinkAbs) == '/dev/null') {
$failed_time = $now - $stat['mtime'];
if ($failed_time > binarypool_config::getBadUrlExpiry()) {
unlink($symlinkAbs);
return array('time' => 0, 'revalidate' => true, 'cache_age' => $failed_time);
}
$failed_nextfetch = $stat['mtime'] + binarypool_config::getBadUrlExpiry() - $now;
throw new binarypool_exception(122, 400, "File download failed {$failed_time} seconds ago. Re-fetching allowed in next time in {$failed_nextfetch} seconds: {$url}");
}
$cache_age = $now - $stat['mtime'];
$revalidate = false;
if ($cache_age > binarypool_config::getCacheRevalidate($bucket)) {
$revalidate = true;
}
return array('time' => filemtime($symlinkAbs), 'revalidate' => $revalidate, 'cache_age' => $cache_age);
}
/**
* Create the URL view symlink for files which were downloaded
*
* @param $asset: An object of binarypool_asset.
*/
private static function linkURL($bucket, $asset, $metadata)
{
if (empty($metadata['URL'])) {
return;
}
$assetDir = '../../' . self::getCleanedBasepath($asset);
$symlink = self::getDownloadedViewPath($bucket, $metadata['URL']);
if (!self::$lastModified) {
self::$lastModified = new binarypool_lastmodified();
}
$lastmodified = self::$lastModified->lastModified($bucket, $metadata['URL']);
if ($lastmodified['cache_age'] > binarypool_config::getCacheRevalidate($bucket)) {
self::refreshLink($bucket, $assetDir, $symlink);
} else {
self::createLink($bucket, $assetDir, $symlink);
}
}
