/** * Convert a picture to a different format and/or resize it. * * @param $from: File path to the original file. * @param $to: File path where the new file should be saved. * @param $mime: MIME type of the original image. * @param $maxWidth: Maximum width of the new image. No resizing * is done if this is empty. Can be used without * specifying maxHeight. * @param $maxHeight: Maximum height of the new image. Must be * used together with $maxWidth. */ protected static function convert($from, $to, $mime, $maxWidth = '', $maxHeight = '') { $cmd = binarypool_config::getUtilityPath('convert'); # Make sure all output images are in RGB. This handles incoming CMYK # images which some browsers can't display. $cmd .= ' -colorspace RGB'; switch ($mime) { case 'image/pdf': case 'application/pdf': $cmd .= ' -trim'; break; case 'image/gif': break; default: $cmd .= ' -flatten'; break; } if ($maxWidth != '') { $scale = intval($maxWidth); if ($maxHeight != '') { $scale .= 'x' . intval($maxHeight); # Don't enlarge if the size is already smaller than resized version $scale .= '>'; } $cmd .= ' -resize ' . escapeshellarg($scale) . ''; } if ($mime == 'image/jpeg') { # Sharpen image $cmd .= ' -unsharp 0.5x1'; } $cmd = $cmd . ' ' . escapeshellarg($from) . ' ' . escapeshellarg($to); $log = new api_log(); $log->debug("Resizing image using command: {$cmd}"); shell_exec($cmd); }
/** * Prepare a curl handle for the http request * * @param String $url the URL to fetch * @param int $modified (optional) seconds since Unix epoch of local copy * @return $curl curl handle */ protected function prepareCurl($url, $lastmodified = 0) { $curl = curl_init(); // See http://curl.haxx.se/libcurl/c/curl_easy_setopt.html curl_setopt($curl, CURLOPT_URL, $url); curl_setopt($curl, CURLOPT_USERAGENT, binarypool_config::getUseragent()); curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1); // return data as string curl_setopt($curl, CURLOPT_FOLLOWLOCATION, 0); // don't follow redirects curl_setopt($curl, CURLOPT_CONNECTTIMEOUT, $this->connectionTimeout); // can't connect? fail fast... curl_setopt($curl, CURLOPT_TIMEOUT, $this->timeout); // total timeout curl_setopt($curl, CURLOPT_NOSIGNAL, true); curl_setopt($curl, CURLOPT_DNS_USE_GLOBAL_CACHE, FALSE); curl_setopt($curl, CURLOPT_FILETIME, TRUE); curl_setopt($curl, CURLOPT_HEADER, TRUE); curl_setopt($curl, CURLOPT_ENCODING, ''); // sends Accept-Encoding for all suported encodings if ($lastmodified) { curl_setopt($curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE); curl_setopt($curl, CURLOPT_TIMEVALUE, $lastmodified); } return $curl; }
protected function touch($bucket, $uri) { $storage = new binarypool_storage($bucket); $assetFile = $uri; if (!$storage->isFile($assetFile)) { $assetFile .= '/index.xml'; if (!$storage->isFile($assetFile)) { return false; } } // Get TTL from request $buckets = binarypool_config::getBuckets(); $ttl = $buckets[$bucket]['ttl']; if ($this->request->getParam('TTL')) { $newttl = intval($this->request->getParam('TTL')); if ($newttl <= $ttl) { // Don't allow higher TTL than bucket configuration $ttl = $newttl; } } // Set TTL $oldAsset = $storage->getAssetObject($assetFile); $asset = $storage->getAssetObject($assetFile); $asset->setExpiry(time() + $ttl * 24 * 60 * 60); $storage->saveAsset($asset, $assetFile); // Update views binarypool_views::updated($bucket, $assetFile, $oldAsset); $this->setResponseCode(204); return true; }
function cleanSymlinks() { // Do symlink cleanup printf("[%10s] Cleaning up symlinks.\n", 'FINAL'); $cmd = binarypool_config::getUtilityPath('symlinks'); system("{$cmd} -cdrs " . binarypool_config::getRoot() . "*/created"); system("{$cmd} -cdrs " . binarypool_config::getRoot() . "*/expiry"); system("{$cmd} -cdrsv " . binarypool_config::getRoot() . "*/downloaded |grep '/dev/null' |xargs -0 rm"); }
/** * Convert a PDF/EPS to JPG/PNG. * * @param $from: File path to the original file. * @param $to: File path where the new file should be saved. * @param $format: Format to convert to. */ protected static function convert($from, $to, $format) { $cmd = binarypool_config::getUtilityPath('pdfconverter'); $cmd .= ' -f ' . $format; $cmd .= ' ' . escapeshellarg($from) . ' ' . escapeshellarg($to); $log = new api_log(); $log->debug("Rendering PDF with command: {$cmd}"); shell_exec($cmd); }
protected function execute() { $xml = "<status method='get'>"; $buckets = binarypool_config::getBuckets(); foreach (array_keys($buckets) as $bucket) { $xml .= '<bucket id="' . htmlspecialchars($bucket) . '" />'; } $xml .= "</status>"; array_push($this->data, new api_model_xml($xml)); }
public function __construct($bucket) { $buckets = binarypool_config::getBuckets(); if (!isset($buckets[$bucket])) { throw new binarypool_exception(100, 404, "Bucket not defined: {$bucket}"); } $this->bucketName = $bucket; $this->bucketConfig = $buckets[$bucket]; $this->storage = $this->getStorage($this->bucketConfig); }
/** * The getExpired call should return an asset which file expired * yesterday. */ function testGetExpiredYesterday() { $assetId = '096dfa489bc3f21df56eded2143843f135ae967e'; $asset = $this->storage->save('IMAGE', array('_' => array('file' => $this->testfile))); $date = date('Y/m/d', strtotime('-1 days')); $viewToday = self::$BUCKET . 'expiry/' . $date . '/' . $assetId; mkdir(dirname($viewToday), 0755, true); symlink(dirname(binarypool_config::getRoot() . $asset), $viewToday); $this->assertEqual(array('test/09/096dfa489bc3f21df56eded2143843f135ae967e/index.xml'), binarypool_browser::getExpired('test')); }
protected static function convert($from, $to) { $cmd = binarypool_config::getUtilityPath('convert'); $cmd .= ' -density 288 -resize 220 +antialias'; # Make sure all output images are in RGB. This handles incoming CMYK # images which some browsers can't display. $cmd .= ' -colorspace RGB -trim '; $cmd .= escapeshellarg($from) . ' ' . escapeshellarg($to); $log = new api_log(); $log->debug("Converting PDF/EPS image using command: {$cmd}"); shell_exec($cmd); }
function setUp() { parent::setUp(); // Remove bucket $bucket = binarypool_config::getRoot() . 'test/'; if (file_exists($bucket)) { $this->deltree($bucket); } // Remove trash if (file_exists(binarypool_config::getRoot() . 'Trash/')) { $this->deltree(binarypool_config::getRoot() . 'Trash/'); } }
/** * Validate original uploaded files and throw exceptions if they are * invalid. * * @param $type: Type - the validation method is chosen based on this type. * @param $bucket: Bucket and type define the validations that apply. * @param $original: Absolute path to the original file. * @return Boolean */ public static function validate($type, $bucket, $original) { $validationClass = "binarypool_validate_" . strtolower($type); if (!class_exists($validationClass)) { return true; } $config = null; $buckets = binarypool_config::getBuckets(); if (isset($buckets[$bucket]['validations']) && isset($buckets[$bucket]['validations'][$type])) { $config = $buckets[$bucket]['validations'][$type]; } return call_user_func($validationClass . '::validate', $original, $config); }
function setUp() { self::$BUCKET = binarypool_config::getRoot() . 'test/'; // Remove bucket if (file_exists(self::$BUCKET)) { $this->deltree(self::$BUCKET); } // Remove trash if (file_exists(binarypool_config::getRoot() . 'Trash/')) { $this->deltree(binarypool_config::getRoot() . 'Trash/'); } $this->testfile = realpath(dirname(__FILE__) . '/res/vw_golf.jpg'); }
protected function execute() { $uri = $this->request->getPath(); // Deletions allowed on bucket? $buckets = binarypool_config::getBuckets(); if (!isset($buckets[$this->bucket]['allowDeletions']) || $buckets[$this->bucket]['allowDeletions'] == false) { throw new binarypool_exception(108, 403, "Deletions not allowed on this bucket."); } $storage = new binarypool_storage($this->bucket); $storage->delete($uri); $this->setResponseCode(204); $this->response->send(); $this->ignoreView = true; }
protected static function lint($source, $xsd = null) { $cmd = binarypool_config::getUtilityPath('xmllint'); $cmd .= ' --nowarning --nonet --noout'; if (!is_null($xsd)) { $cmd .= ' --schema ' . escapeshellarg($xsd); } $cmd = $cmd . ' ' . escapeshellarg($source) . ' >/dev/null 2>/dev/null'; system($cmd, $retval); if ($retval != 0) { if (is_null($xsd)) { throw new binarypool_exception(117, 400, "XML document is not well-formed."); } else { throw new binarypool_exception(118, 400, "XML document is not valid."); } } return true; }
function walk_dir($root, $callback, $exclude = array()) { $fsroot = binarypool_config::getRoot(); $root = rtrim($root, '/') . '/'; $queue = array($root); foreach ($exclude as &$path) { $path = $root . trim($path, '/') . '/'; } while ($base = array_shift($queue)) { $relative = substr($base, strlen($fsroot)); $callback($relative); if ($handle = opendir($base)) { while (($child = readdir($handle)) !== FALSE) { if (is_dir($base . $child) && $child != '.' && $child != '..') { $combined_path = $base . $child . '/'; if (!in_array($combined_path, $exclude)) { array_push($queue, $combined_path); } } } closedir($handle); } } }
/** * Create an asset XML with an absolute base path and reload it. */ function testAssetPersistanceWithAbsoluteBasePath() { $basepath = 'test/somehashhere/'; $absBasepath = binarypool_config::getRoot() . $basepath; mkdir($absBasepath, 0755, true); $asset = $this->testAssetWithAbsoluteBasePath(); file_put_contents($absBasepath . 'index.xml', $asset->getXML()); // Load XML from FS $asset = new binarypool_asset($this->getDummyStorage(), $basepath . 'index.xml'); $xml = $asset->getXML(); $dom = DOMDocument::loadXML($xml); $xp = new DOMXPath($dom); // Test that the same information is still around $this->assertXPath($xp, '/registry/@version', '3.0'); $this->assertXPath($xp, '/registry/items/item/location', 'http://bin.staticlocal.ch/vw_golf.jpg'); $this->assertXPath($xp, '/registry/items/item/location/@absolute', 'true'); $this->assertXPath($xp, '/registry/items/item/mimetype', 'image/jpeg'); $this->assertXPath($xp, '/registry/items/item/size', '51941'); return $asset; }
/** * Uses `file' on the command line to get the MIME type. */ protected static function getMimeTypeWithCmdLineFile($file) { $cmd = binarypool_config::getUtilityPath('file'); $mime = trim(shell_exec("{$cmd} -ib " . escapeshellarg($file))); if ($mime == '' || $mime == 'regular file') { return null; } else { return $mime; } }
/** * Fix the extension of manually uploaded renditions. */ function testSaveRenditionFixExtension() { $this->assertFalse(file_exists(self::$BUCKET . 'cb/cbf9f9f453acaba556e00b48951815da5611f975/vw_golf_smaller.jpg')); $asset = $this->storage->save('IMAGE', array('_' => array('file' => dirname(__FILE__) . '/../res/vw_golf_smaller.jpg'), 'detailpage' => array('file' => dirname(__FILE__) . '/../res/upload1.bin'))); $this->assertNotNull($asset); $this->assertEqual('test/cb/cbf9f9f453acaba556e00b48951815da5611f975/index.xml', $asset); $this->assertTrue(file_exists(self::$BUCKET . 'cb/cbf9f9f453acaba556e00b48951815da5611f975/vw_golf_smaller.jpg'), 'Original file was not written to file system.'); $this->assertTrue(file_exists(self::$BUCKET . 'cb/cbf9f9f453acaba556e00b48951815da5611f975/upload1.jpg'), 'detailpage rendition file was not written to file system.'); $this->assertFalse(file_exists(self::$BUCKET . 'cb/cbf9f9f453acaba556e00b48951815da5611f975/upload1.bin'), 'detailpage rendition file was written to file system with the wrong extension.'); $this->assertTrue(file_exists(binarypool_config::getRoot() . $asset), 'Asset file was not written to file system.'); }
protected function getFilesFromUrl() { $url = $this->request->getParam('URL'); $this->log->debug("Downloading file: %s", $url); if (!self::$lastModified) { self::$lastModified = new binarypool_lastmodified(); } $lastmodified = self::$lastModified->lastModified($this->bucket, $url); if (binarypool_config::getCacheRevalidate($this->bucket) === 0) { $lastmodified['time'] = 0; } $tmpfile = tempnam(sys_get_temp_dir(), 'binary'); if ($tmpfile == '' || $tmpfile === FALSE) { throw new binarypool_exception(104, 500, "Could not create temporary file"); } array_push($this->tmpfiles, $tmpfile); $result = array('code' => 0, 'headers' => array(), 'body' => ''); $retries = 3; if ($lastmodified['revalidate']) { $httpc = new binarypool_httpclient(); while ($retries) { try { $result = $httpc->download($url, $tmpfile, $lastmodified['time']); if ($result['code'] < 500) { break; } } catch (binarypool_httpclient_exception $e) { // ignore - dropped connections etc. - retry $this->log->debug("Failed download attempt from %s: %s", $url, $e); } sleep(1); $retries--; } } else { $result['code'] = 304; } if (304 == $result['code']) { $this->log->debug("File %s has not been modified", $url); return array(); } if ($result['code'] != 200 || !filesize($tmpfile)) { binarypool_views::flagBadUrl($this->bucket, $url); throw new binarypool_exception(121, 400, "File could not be fetched from URL: " . $url); } $url_parsed = parse_url($url); $filename = basename($url_parsed['path']); # Restrict filenames TO ALPHANUMS and reduce sequences of '.' to avoid # traversal issues, unicode issues, command injection etc. $filename = preg_replace(array('#\\.{2,}#', '#[^a-zA-Z0-9\\.]+#'), array('.', '_'), $filename); return array('_' => array('file' => $tmpfile, 'filename' => $filename)); }
/** * Load settings from the config file. */ private static function load() { // Read correct config file based on environment $env = ''; if (isset($_SERVER['BINARYPOOL_CONFIG'])) { $env = $_SERVER['BINARYPOOL_CONFIG']; } $filename = API_PROJECT_DIR . "/conf/binarypool.php"; if ($env != '' && file_exists($filename)) { $filename = API_PROJECT_DIR . '/conf/binarypool-' . $env . '.php'; } include $filename; // Get values self::$buckets = $BUCKETS; self::$root = $ROOT; if (!file_exists(self::$root)) { mkdir(self::$root); if (!file_exists(self::$root)) { throw new binarypool_exception(107, 500, "Binary Pool path is not available."); } } self::$root = realpath(self::$root); if (self::$root[strlen(self::$root) - 1] !== '/') { self::$root .= '/'; } self::$paths = $PATHS; self::$badUrlExpiry = $BADURLEXPIRY; self::$cacheRevalidate = $CACHEREVALIDATE; self::$useragent = $USERAGENT; }
protected function getS3Bucket() { $buckets = binarypool_config::getBuckets(); return $buckets['test_s3']; }
/** * Calculates the renditions for the given file. * * @param $type: Type - the renderer is chosen based on this type. * @param $bucket: Bucket and type define the renditions which have * to be generated. * @param $original: Absolute path to the original file. * @param $outputPath: Directory where the renditions are written to. * @param $exclude: Renditions which don't have to be calculated. * @param $assetFile: Path to the asset file. Needed for asynchronous * rendition processes. * @return Associative array of hashes. The key is the rendition name, * the path is the relative path of the file. */ public static function render($type, $bucket, $original, $outputPath, $exclude = array(), $assetFile = null) { // Get configured renditions for this type $buckets = binarypool_config::getBuckets(); $renditions = array(); if (!isset($buckets[$bucket]['renditions'][$type])) { return $renditions; } // Information about the original file $mime = binarypool_mime::getMimeType($original); // Loops through the renditions configuration and generates the // renditions as specified. // // The following keys of each rendition config are considered: // - _sources: Ordered array of rendition names that should be // used as input for this rendition. If not present // of empty elements mean that the original // binary is used. // - _class: The render_base subclass to use. The class is // prefixed with 'binarypool_render_' to get the // definitive PHP class name. // - _mimes: MIME type of the original binary for which this // rendition is calculated. // // This three keys are removed from the array and the modified // array is then passed to the render class as configuration. // // The list of renditions needs to be in the correct dependency // order. If rendition `B' uses rendition `A' as a source, then // rendition `A' must come before `B'. foreach ($buckets[$bucket]['renditions'][$type] as $rendition => $renditionConfig) { // Renditions that the user uploaded are not calculated if (in_array($rendition, $exclude)) { continue; } // Rendition to be calculated? if (isset($renditionConfig['_mimes'])) { $mimesCfg = $renditionConfig['_mimes']; if (!is_array($mimesCfg)) { $mimesCfg = array($mimesCfg); } if (!in_array($mime, $mimesCfg)) { continue; } unset($renditionConfig['_mimes']); } // Get correct source file name $sourceFile = null; if (isset($renditionConfig['_sources'])) { foreach ($renditionConfig['_sources'] as $sourceRendition) { if (isset($renditions[$sourceRendition])) { $sourceFile = $renditions[$sourceRendition]; break; } else { if ($sourceRendition === '') { $sourceFile = $original; break; } } } unset($renditionConfig['_sources']); } else { $sourceFile = $original; } if (is_null($sourceFile)) { throw new binarypool_exception(106, 500, "Missing source rendition for rendition {$rendition}"); } $renditionConfig['_bucket'] = $bucket; // Get correct class name $renditionType = strtolower($type); if (isset($renditionConfig['_class'])) { $renditionType = $renditionConfig['_class']; unset($renditionConfig['_class']); } $renderingClass = "binarypool_render_" . $renditionType; // Filenames $renditionFile = $rendition; $absoluteRendition = $outputPath . $renditionFile; // Render image. Return value of rendering is the absolute path to the // rendition including file extensions. $absoluteRendition = call_user_func($renderingClass . '::render', $sourceFile, $absoluteRendition, $assetFile, $renditionConfig); if (!is_null($absoluteRendition)) { if (!file_exists($absoluteRendition)) { throw new binarypool_exception(106, 500, "Could not create rendition: {$rendition}"); } $renditions[$rendition] = $absoluteRendition; } } return $renditions; }
function testGetRenditionDirectory() { $storage = new binarypool_storage_driver_file(); $this->assertEqual($storage->getRenditionsDirectory('files/abc/def'), binarypool_config::getRoot() . 'files/abc/def/'); }
function testGetPathOfConvert() { $convert = binarypool_config::getUtilityPath('convert'); $this->assertNotEqual(0, strlen($convert)); }
function testUpdatedSymlink() { $this->assignLastModified(); $cache_age = binarypool_config::getCacheRevalidate('test') + 1; binarypool_views::$lastModified->setReturnValue('lastModified', array('cache_age' => $cache_age)); $asset = $this->createMockAsset(); $storage = $this->createMockStorage($asset); $storage->expectCallCount('symlink', 2); $storage->expectCallCount('relink', 1); $this->assignMockStorageFactory($storage); binarypool_views::created('test', 'foo', array('URL' => 'http://local.ch/foo.gif')); }
/** * Tests that the correct rendition is saved. */ function testUploadRenditionVerify() { $this->testUploadRendition(); $this->get('/test/cb/cbf9f9f453acaba556e00b48951815da5611f975/index.xml'); $this->assertEqual(api_response::getInstance()->getCode(), 200); $localHash = sha1_file(dirname(__FILE__) . '/../res/vw_golf_blur.jpg'); $remoteFile = $this->getText('/registry/items/item[rendition="detailpage"]/location'); $remoteHash = sha1_file(binarypool_config::getRoot() . '/' . $remoteFile); $this->assertEqual($localHash, $remoteHash); }
public function getURLLastModified($url, $symlink, $bucket) { $symlink .= '.link'; $now = $this->time; if (!$this->fileExists($symlink)) { return array('time' => 0, 'revalidate' => true, 'cache_age' => 0); } $contents = $this->getFile($symlink); $contents = json_decode($contents, true); if ($contents['link'] == '/dev/null') { // Dead URL $failed_time = $now - $contents['mtime']; if ($failed_time > binarypool_config::getBadUrlExpiry()) { $this->unlink($symlink); return array('time' => 0, 'revalidate' => true, 'cache_age' => $failed_time); } $failed_nextfetch = $contents['mtime'] + binarypool_config::getBadUrlExpiry() - $now; throw new binarypool_exception(122, 400, "File download failed {$failed_time} seconds ago. Re-fetching allowed in next time in {$failed_nextfetch} seconds: {$url}"); } $cache_age = $now - $contents['mtime']; $revalidate = false; if ($cache_age > binarypool_config::getCacheRevalidate($bucket)) { $revalidate = true; } return array('time' => $contents['mtime'], 'revalidate' => $revalidate, 'cache_age' => $cache_age); }
public function getURLLastModified($url, $symlink, $bucket) { $this->clearstatcache(); if (!$this->fileExists($symlink)) { return array('time' => 0, 'revalidate' => true, 'cache_age' => 0); } $symlinkAbs = $this->absolutize($symlink); $stat = lstat($symlinkAbs); $now = time(); if (readlink($symlinkAbs) == '/dev/null') { $failed_time = $now - $stat['mtime']; if ($failed_time > binarypool_config::getBadUrlExpiry()) { unlink($symlinkAbs); return array('time' => 0, 'revalidate' => true, 'cache_age' => $failed_time); } $failed_nextfetch = $stat['mtime'] + binarypool_config::getBadUrlExpiry() - $now; throw new binarypool_exception(122, 400, "File download failed {$failed_time} seconds ago. Re-fetching allowed in next time in {$failed_nextfetch} seconds: {$url}"); } $cache_age = $now - $stat['mtime']; $revalidate = false; if ($cache_age > binarypool_config::getCacheRevalidate($bucket)) { $revalidate = true; } return array('time' => filemtime($symlinkAbs), 'revalidate' => $revalidate, 'cache_age' => $cache_age); }
/** * Create the URL view symlink for files which were downloaded * * @param $asset: An object of binarypool_asset. */ private static function linkURL($bucket, $asset, $metadata) { if (empty($metadata['URL'])) { return; } $assetDir = '../../' . self::getCleanedBasepath($asset); $symlink = self::getDownloadedViewPath($bucket, $metadata['URL']); if (!self::$lastModified) { self::$lastModified = new binarypool_lastmodified(); } $lastmodified = self::$lastModified->lastModified($bucket, $metadata['URL']); if ($lastmodified['cache_age'] > binarypool_config::getCacheRevalidate($bucket)) { self::refreshLink($bucket, $assetDir, $symlink); } else { self::createLink($bucket, $assetDir, $symlink); } }