/**
* Returns the most appropriate source image for the thumbnail, given a target thumbnail size
* @param array $params
* @return array Source path and width/height of the source
*/
public function getThumbnailSource($params)
{
if ($this->repo && $this->getHandler()->supportsBucketing() && isset($params['physicalWidth']) && ($bucket = $this->getThumbnailBucket($params['physicalWidth']))) {
if ($this->getWidth() != 0) {
$bucketHeight = round($this->getHeight() * ($bucket / $this->getWidth()));
} else {
$bucketHeight = 0;
}
// Try to avoid reading from storage if the file was generated by this script
if (isset($this->tmpBucketedThumbCache[$bucket])) {
$tmpPath = $this->tmpBucketedThumbCache[$bucket];
if (file_exists($tmpPath)) {
return array('path' => $tmpPath, 'width' => $bucket, 'height' => $bucketHeight);
}
}
$bucketPath = $this->getBucketThumbPath($bucket);
if ($this->repo->fileExists($bucketPath)) {
$fsFile = $this->repo->getLocalReference($bucketPath);
if ($fsFile) {
return array('path' => $fsFile->getPath(), 'width' => $bucket, 'height' => $bucketHeight);
}
}
}
// Thumbnailing a very large file could result in network saturation if
// everyone does it at once.
if ($this->getSize() >= 10000000.0) {
// 10MB
$that = $this;
$work = new PoolCounterWorkViaCallback('GetLocalFileCopy', sha1($this->getName()), array('doWork' => function () use($that) {
return $that->getLocalRefPath();
}));
$srcPath = $work->execute();
} else {
$srcPath = $this->getLocalRefPath();
}
// Original file
return array('path' => $srcPath, 'width' => $this->getWidth(), 'height' => $this->getHeight());
}
/**
* @param File $image
* @param string $dstPath
* @param string $dstUrl
* @param array $params
* @param int $flags
* @return MediaTransformError|ThumbnailImage|TransformParameterError
*/
function doTransform($image, $dstPath, $dstUrl, $params, $flags = 0)
{
global $wgDjvuRenderer, $wgDjvuPostProcessor;
// Fetch XML and check it, to give a more informative error message than the one which
// normaliseParams will inevitably give.
$xml = $image->getMetadata();
if (!$xml) {
$width = isset($params['width']) ? $params['width'] : 0;
$height = isset($params['height']) ? $params['height'] : 0;
return new MediaTransformError('thumbnail_error', $width, $height, wfMessage('djvu_no_xml')->text());
}
if (!$this->normaliseParams($image, $params)) {
return new TransformParameterError($params);
}
$width = $params['width'];
$height = $params['height'];
$page = $params['page'];
if ($page > $this->pageCount($image)) {
return new MediaTransformError('thumbnail_error', $width, $height, wfMessage('djvu_page_error')->text());
}
if ($flags & self::TRANSFORM_LATER) {
$params = array('width' => $width, 'height' => $height, 'page' => $page);
return new ThumbnailImage($image, $dstUrl, $dstPath, $params);
}
if (!wfMkdirParents(dirname($dstPath), null, __METHOD__)) {
return new MediaTransformError('thumbnail_error', $width, $height, wfMessage('thumbnail_dest_directory')->text());
}
// Get local copy source for shell scripts
// Thumbnail extraction is very inefficient for large files.
// Provide a way to pool count limit the number of downloaders.
if ($image->getSize() >= 10000000.0) {
// 10MB
$work = new PoolCounterWorkViaCallback('GetLocalFileCopy', sha1($image->getName()), array('doWork' => function () use($image) {
return $image->getLocalRefPath();
}));
$srcPath = $work->execute();
} else {
$srcPath = $image->getLocalRefPath();
}
if ($srcPath === false) {
// Failed to get local copy
wfDebugLog('thumbnail', sprintf('Thumbnail failed on %s: could not get local copy of "%s"', wfHostname(), $image->getName()));
return new MediaTransformError('thumbnail_error', $params['width'], $params['height'], wfMessage('filemissing')->text());
}
# Use a subshell (brackets) to aggregate stderr from both pipeline commands
# before redirecting it to the overall stdout. This works in both Linux and Windows XP.
$cmd = '(' . wfEscapeShellArg($wgDjvuRenderer, "-format=ppm", "-page={$page}", "-size={$params['physicalWidth']}x{$params['physicalHeight']}", $srcPath);
if ($wgDjvuPostProcessor) {
$cmd .= " | {$wgDjvuPostProcessor}";
}
$cmd .= ' > ' . wfEscapeShellArg($dstPath) . ') 2>&1';
wfProfileIn('ddjvu');
wfDebug(__METHOD__ . ": {$cmd}\n");
$retval = '';
$err = wfShellExec($cmd, $retval);
wfProfileOut('ddjvu');
$removed = $this->removeBadFile($dstPath, $retval);
if ($retval != 0 || $removed) {
$this->logErrorForExternalProcess($retval, $err, $cmd);
return new MediaTransformError('thumbnail_error', $width, $height, $err);
} else {
$params = array('width' => $width, 'height' => $height, 'page' => $page);
return new ThumbnailImage($image, $dstUrl, $dstPath, $params);
}
}
/**
* Actually try to generate a new thumbnail
*
* @param File $file
* @param array $params
* @param string $thumbName
* @param string $thumbPath
* @return array (MediaTransformOutput|bool, string|bool error message HTML)
*/
function wfGenerateThumbnail(File $file, array $params, $thumbName, $thumbPath)
{
global $wgMemc, $wgAttemptFailureEpoch;
$key = wfMemcKey('attempt-failures', $wgAttemptFailureEpoch, $file->getRepo()->getName(), $file->getSha1(), md5($thumbName));
// Check if this file keeps failing to render
if ($wgMemc->get($key) >= 4) {
return array(false, wfMessage('thumbnail_image-failure-limit', 4));
}
$done = false;
// Record failures on PHP fatals in addition to caching exceptions
register_shutdown_function(function () use(&$done, $key) {
if (!$done) {
// transform() gave a fatal
global $wgMemc;
// Randomize TTL to reduce stampedes
$wgMemc->incrWithInit($key, 3600 + mt_rand(0, 300));
}
});
$thumb = false;
$errorHtml = false;
// guard thumbnail rendering with PoolCounter to avoid stampedes
// expensive files use a separate PoolCounter config so it is possible
// to set up a global limit on them
if ($file->isExpensiveToThumbnail()) {
$poolCounterType = 'FileRenderExpensive';
} else {
$poolCounterType = 'FileRender';
}
// Thumbnail isn't already there, so create the new thumbnail...
try {
$work = new PoolCounterWorkViaCallback($poolCounterType, sha1($file->getName()), array('doWork' => function () use($file, $params) {
return $file->transform($params, File::RENDER_NOW);
}, 'getCachedWork' => function () use($file, $params, $thumbPath) {
// If the worker that finished made this thumbnail then use it.
// Otherwise, it probably made a different thumbnail for this file.
return $file->getRepo()->fileExists($thumbPath) ? $file->transform($params, File::RENDER_NOW) : false;
// retry once more in exclusive mode
}, 'fallback' => function () {
return wfMessage('generic-pool-error')->parse();
}, 'error' => function (Status $status) {
return $status->getHTML();
}));
$result = $work->execute();
if ($result instanceof MediaTransformOutput) {
$thumb = $result;
} elseif (is_string($result)) {
// error
$errorHtml = $result;
}
} catch (Exception $e) {
// Tried to select a page on a non-paged file?
}
/** @noinspection PhpUnusedLocalVariableInspection */
$done = true;
// no PHP fatal occured
if (!$thumb || $thumb->isError()) {
// Randomize TTL to reduce stampedes
$wgMemc->incrWithInit($key, 3600 + mt_rand(0, 300));
}
return array($thumb, $errorHtml);
}
/**
* Fetch the translators for a language with caching
*
* @param string $code
* @return array|bool Map of (user name => page count) or false on failure
*/
public function fetchTranslators($code)
{
$cache = wfGetCache(CACHE_ANYTHING);
$cachekey = wfMemcKey('translate-supportedlanguages-translator-list-v1', $code);
if ($this->purge) {
$cache->delete($cachekey);
$data = false;
} else {
$staleCutoffUnix = time() - 3600;
$data = $cache->get($cachekey);
if (is_array($data) && $data['asOfTime'] > $staleCutoffUnix) {
return $data['users'];
}
}
$that = $this;
$work = new PoolCounterWorkViaCallback('TranslateFetchTranslators', "TranslateFetchTranslators-{$code}", array('doWork' => function () use($that, $code, $cache, $cachekey) {
$users = $that->loadTranslators($code);
$newData = array('users' => $users, 'asOfTime' => time());
$cache->set($cachekey, $newData, 86400);
return $users;
}, 'doCachedWork' => function () use($cache, $cachekey) {
$newData = $cache->get($cachekey);
// Use new cache value from other thread
return is_array($newData) ? $newData['users'] : false;
}, 'fallback' => function () use($data) {
// Use stale cache if possible
return is_array($data) ? $data['users'] : false;
}));
return $work->execute();
}
/**
* @param $image File
* @param $dstPath string
* @param $dstUrl string
* @param $params array
* @param $flags int
* @return MediaTransformError|MediaTransformOutput|ThumbnailImage|TransformParameterError
*/
function doTransform($image, $dstPath, $dstUrl, $params, $flags = 0)
{
global $wgPdfProcessor, $wgPdfPostProcessor, $wgPdfHandlerDpi, $wgPdfHandlerJpegQuality;
$metadata = $image->getMetadata();
if (!$metadata) {
return $this->doThumbError(isset($params['width']) ? $params['width'] : null, isset($params['height']) ? $params['height'] : null, 'pdf_no_metadata');
}
if (!$this->normaliseParams($image, $params)) {
return new TransformParameterError($params);
}
$width = $params['width'];
$height = $params['height'];
$page = $params['page'];
if ($page > $this->pageCount($image)) {
return $this->doThumbError($width, $height, 'pdf_page_error');
}
if ($flags & self::TRANSFORM_LATER) {
return new ThumbnailImage($image, $dstUrl, $width, $height, false, $page);
}
if (!wfMkdirParents(dirname($dstPath), null, __METHOD__)) {
return $this->doThumbError($width, $height, 'thumbnail_dest_directory');
}
// Thumbnail extraction is very inefficient for large files.
// Provide a way to pool count limit the number of downloaders.
if ($image->getSize() >= 10000000.0) {
// 10MB
$work = new PoolCounterWorkViaCallback('GetLocalFileCopy', sha1($image->getName()), array('doWork' => function () use($image) {
return $image->getLocalRefPath();
}));
$srcPath = $work->execute();
} else {
$srcPath = $image->getLocalRefPath();
}
if ($srcPath === false) {
// could not download original
return $this->doThumbError($width, $height, 'filemissing');
}
$cmd = '(' . wfEscapeShellArg($wgPdfProcessor, "-sDEVICE=jpeg", "-sOutputFile=-", "-dFirstPage={$page}", "-dLastPage={$page}", "-r{$wgPdfHandlerDpi}", "-dBATCH", "-dNOPAUSE", "-q", $srcPath);
$cmd .= " | " . wfEscapeShellArg($wgPdfPostProcessor, "-depth", "8", "-quality", $wgPdfHandlerJpegQuality, "-resize", $width, "-", $dstPath);
$cmd .= ")";
wfProfileIn('PdfHandler');
wfDebug(__METHOD__ . ": {$cmd}\n");
$retval = '';
$err = wfShellExecWithStderr($cmd, $retval);
wfProfileOut('PdfHandler');
$removed = $this->removeBadFile($dstPath, $retval);
if ($retval != 0 || $removed) {
wfDebugLog('thumbnail', sprintf('thumbnail failed on %s: error %d "%s" from "%s"', wfHostname(), $retval, trim($err), $cmd));
return new MediaTransformError('thumbnail_error', $width, $height, $err);
} else {
return new ThumbnailImage($image, $dstUrl, $width, $height, $dstPath, $page);
}
}
/**
* Generate a diff, no caching
*
* @todo move this to TextDifferenceEngine, make DifferenceEngine abstract. At some point.
*
* @param string $otext Old text, must be already segmented
* @param string $ntext New text, must be already segmented
*
* @return bool|string
*/
public function generateTextDiffBody($otext, $ntext)
{
$diff = function () use($otext, $ntext) {
$time = microtime(true);
$result = $this->textDiff($otext, $ntext);
$time = intval((microtime(true) - $time) * 1000);
$this->getStats()->timing('diff_time', $time);
// Log requests slower than 99th percentile
if ($time > 100 && $this->mOldPage && $this->mNewPage) {
wfDebugLog('diff', "{$time} ms diff: {$this->mOldid} -> {$this->mNewid} {$this->mNewPage}");
}
return $result;
};
$error = function ($status) {
throw new FatalError($status->getWikiText());
};
// Use PoolCounter if the diff looks like it can be expensive
if (strlen($otext) + strlen($ntext) > 20000) {
$work = new PoolCounterWorkViaCallback('diff', md5($otext) . md5($ntext), ['doWork' => $diff, 'error' => $error]);
return $work->execute();
}
return $diff();
}
/**
* @param $image File
* @return bool
*/
function getMetaArray($image)
{
if (isset($image->pdfMetaArray)) {
return $image->pdfMetaArray;
}
$metadata = $image->getMetadata();
if (!$this->isMetadataValid($image, $metadata)) {
wfDebug("Pdf metadata is invalid or missing, should have been fixed in upgradeRow\n");
return false;
}
$work = new PoolCounterWorkViaCallback('PdfHandler-unserialize-metadata', $image->getName(), array('doWork' => function () use($image, $metadata) {
wfSuppressWarnings();
$image->pdfMetaArray = unserialize($metadata);
wfRestoreWarnings();
}));
$work->execute();
return $image->pdfMetaArray;
}
/**
* Generate a diff, no caching
*
* @todo move this to TextDifferenceEngine, make DifferenceEngine abstract. At some point.
*
* @param string $otext Old text, must be already segmented
* @param string $ntext New text, must be already segmented
*
* @return bool|string
*/
public function generateTextDiffBody($otext, $ntext)
{
$self = $this;
$diff = function () use($self, $otext, $ntext) {
return $self->textDiff($otext, $ntext);
};
$error = function ($status) {
throw new FatalError($status->getWikiText());
};
// Use PoolCounter if the diff looks like it can be expensive
if (strlen($otext) + strlen($ntext) > 20000) {
$work = new PoolCounterWorkViaCallback('diff', md5($otext) . md5($ntext), array('doWork' => $diff, 'error' => $error));
return $work->execute();
}
return $diff();
}
