/**
* Return an array of files where the name starts with $prefix.
*
* @param string $prefix The prefix to search for
* @param int $limit The maximum amount of files to return
* @return array
*/
public function findFilesByPrefix($prefix, $limit)
{
$selectOptions = array('ORDER BY' => 'img_name', 'LIMIT' => intval($limit));
// Query database
$dbr = $this->getSlaveDB();
$res = $dbr->select('image', LocalFile::selectFields(), 'img_name ' . $dbr->buildLike($prefix, $dbr->anyString()), __METHOD__, $selectOptions);
// Build file objects
$files = array();
foreach ($res as $row) {
$files[] = $this->newFileFromRow($row);
}
return $files;
}
/**
* Get an array or iterator of file objects for files that have a given
* SHA-1 content hash.
*/
function findBySha1($hash)
{
$dbr = $this->getSlaveDB();
$res = $dbr->select('image', LocalFile::selectFields(), array('img_sha1' => $hash));
$result = array();
while ($row = $res->fetchObject()) {
$result[] = $this->newFileFromRow($row);
}
$res->free();
return $result;
}
function findFiles($titles)
{
// FIXME: Only accepts a $titles array where the keys are the sanitized
// file names.
if (count($titles) == 0) {
return array();
}
$dbr = $this->getSlaveDB();
$res = $dbr->select('image', LocalFile::selectFields(), array('img_name' => array_keys($titles)));
$result = array();
while ($row = $res->fetchObject()) {
$result[$row->img_name] = $this->newFileFromRow($row);
}
$res->free();
return $result;
}
/**
* @param ApiPageSet $resultPageSet
* @return void
*/
private function run($resultPageSet = null)
{
$repo = $this->mRepo;
if (!$repo instanceof LocalRepo) {
$this->dieUsage('Local file repository does not support querying all images', 'unsupportedrepo');
}
$prefix = $this->getModulePrefix();
$db = $this->getDB();
$params = $this->extractRequestParams();
// Table and return fields
$this->addTables('image');
$prop = array_flip($params['prop']);
$this->addFields(LocalFile::selectFields());
$ascendingOrder = true;
if ($params['dir'] == 'descending' || $params['dir'] == 'older') {
$ascendingOrder = false;
}
if ($params['sort'] == 'name') {
// Check mutually exclusive params
$disallowed = ['start', 'end', 'user'];
foreach ($disallowed as $pname) {
if (isset($params[$pname])) {
$this->dieUsage("Parameter '{$prefix}{$pname}' can only be used with {$prefix}sort=timestamp", 'badparams');
}
}
if ($params['filterbots'] != 'all') {
$this->dieUsage("Parameter '{$prefix}filterbots' can only be used with {$prefix}sort=timestamp", 'badparams');
}
// Pagination
if (!is_null($params['continue'])) {
$cont = explode('|', $params['continue']);
$this->dieContinueUsageIf(count($cont) != 1);
$op = $ascendingOrder ? '>' : '<';
$continueFrom = $db->addQuotes($cont[0]);
$this->addWhere("img_name {$op}= {$continueFrom}");
}
// Image filters
$from = $params['from'] === null ? null : $this->titlePartToKey($params['from'], NS_FILE);
$to = $params['to'] === null ? null : $this->titlePartToKey($params['to'], NS_FILE);
$this->addWhereRange('img_name', $ascendingOrder ? 'newer' : 'older', $from, $to);
if (isset($params['prefix'])) {
$this->addWhere('img_name' . $db->buildLike($this->titlePartToKey($params['prefix'], NS_FILE), $db->anyString()));
}
} else {
// Check mutually exclusive params
$disallowed = ['from', 'to', 'prefix'];
foreach ($disallowed as $pname) {
if (isset($params[$pname])) {
$this->dieUsage("Parameter '{$prefix}{$pname}' can only be used with {$prefix}sort=name", 'badparams');
}
}
if (!is_null($params['user']) && $params['filterbots'] != 'all') {
// Since filterbots checks if each user has the bot right, it
// doesn't make sense to use it with user
$this->dieUsage("Parameters '{$prefix}user' and '{$prefix}filterbots' cannot be used together", 'badparams');
}
// Pagination
$this->addTimestampWhereRange('img_timestamp', $ascendingOrder ? 'newer' : 'older', $params['start'], $params['end']);
// Include in ORDER BY for uniqueness
$this->addWhereRange('img_name', $ascendingOrder ? 'newer' : 'older', null, null);
if (!is_null($params['continue'])) {
$cont = explode('|', $params['continue']);
$this->dieContinueUsageIf(count($cont) != 2);
$op = $ascendingOrder ? '>' : '<';
$continueTimestamp = $db->addQuotes($db->timestamp($cont[0]));
$continueName = $db->addQuotes($cont[1]);
$this->addWhere("img_timestamp {$op} {$continueTimestamp} OR " . "(img_timestamp = {$continueTimestamp} AND " . "img_name {$op}= {$continueName})");
}
// Image filters
if (!is_null($params['user'])) {
$this->addWhereFld('img_user_text', $params['user']);
}
if ($params['filterbots'] != 'all') {
$this->addTables('user_groups');
$this->addJoinConds(['user_groups' => ['LEFT JOIN', ['ug_group' => User::getGroupsWithPermission('bot'), 'ug_user = img_user']]]);
$groupCond = $params['filterbots'] == 'nobots' ? 'NULL' : 'NOT NULL';
$this->addWhere("ug_group IS {$groupCond}");
}
}
// Filters not depending on sort
if (isset($params['minsize'])) {
$this->addWhere('img_size>=' . intval($params['minsize']));
}
if (isset($params['maxsize'])) {
$this->addWhere('img_size<=' . intval($params['maxsize']));
}
$sha1 = false;
if (isset($params['sha1'])) {
$sha1 = strtolower($params['sha1']);
if (!$this->validateSha1Hash($sha1)) {
$this->dieUsage('The SHA1 hash provided is not valid', 'invalidsha1hash');
}
$sha1 = Wikimedia\base_convert($sha1, 16, 36, 31);
} elseif (isset($params['sha1base36'])) {
$sha1 = strtolower($params['sha1base36']);
if (!$this->validateSha1Base36Hash($sha1)) {
$this->dieUsage('The SHA1Base36 hash provided is not valid', 'invalidsha1base36hash');
}
}
if ($sha1) {
$this->addWhereFld('img_sha1', $sha1);
}
if (!is_null($params['mime'])) {
if ($this->getConfig()->get('MiserMode')) {
$this->dieUsage('MIME search disabled in Miser Mode', 'mimesearchdisabled');
}
$mimeConds = [];
foreach ($params['mime'] as $mime) {
list($major, $minor) = File::splitMime($mime);
$mimeConds[] = $db->makeList(['img_major_mime' => $major, 'img_minor_mime' => $minor], LIST_AND);
}
// safeguard against internal_api_error_DBQueryError
if (count($mimeConds) > 0) {
$this->addWhere($db->makeList($mimeConds, LIST_OR));
} else {
// no MIME types, no files
$this->getResult()->addValue('query', $this->getModuleName(), []);
return;
}
}
$limit = $params['limit'];
$this->addOption('LIMIT', $limit + 1);
$sortFlag = '';
if (!$ascendingOrder) {
$sortFlag = ' DESC';
}
if ($params['sort'] == 'timestamp') {
$this->addOption('ORDER BY', 'img_timestamp' . $sortFlag);
if (!is_null($params['user'])) {
$this->addOption('USE INDEX', ['image' => 'img_usertext_timestamp']);
} else {
$this->addOption('USE INDEX', ['image' => 'img_timestamp']);
}
} else {
$this->addOption('ORDER BY', 'img_name' . $sortFlag);
}
$res = $this->select(__METHOD__);
$titles = [];
$count = 0;
$result = $this->getResult();
foreach ($res as $row) {
if (++$count > $limit) {
// We've reached the one extra which shows that there are
// additional pages to be had. Stop here...
if ($params['sort'] == 'name') {
$this->setContinueEnumParameter('continue', $row->img_name);
} else {
$this->setContinueEnumParameter('continue', "{$row->img_timestamp}|{$row->img_name}");
}
break;
}
if (is_null($resultPageSet)) {
$file = $repo->newFileFromRow($row);
$info = array_merge(['name' => $row->img_name], ApiQueryImageInfo::getInfo($file, $prop, $result));
self::addTitleInfo($info, $file->getTitle());
$fit = $result->addValue(['query', $this->getModuleName()], null, $info);
if (!$fit) {
if ($params['sort'] == 'name') {
$this->setContinueEnumParameter('continue', $row->img_name);
} else {
$this->setContinueEnumParameter('continue', "{$row->img_timestamp}|{$row->img_name}");
}
break;
}
} else {
$titles[] = Title::makeTitle(NS_FILE, $row->img_name);
}
}
if (is_null($resultPageSet)) {
$result->addIndexedTagName(['query', $this->getModuleName()], 'img');
} else {
$resultPageSet->populateFromTitles($titles);
}
}
/**
* Get an array of arrays or iterators of file objects for files that
* have the given SHA-1 content hashes.
*
* Overrides generic implementation in FileRepo for performance reason
*
* @param $hashes array An array of hashes
* @return array An Array of arrays or iterators of file objects and the hash as key
*/
function findBySha1s(array $hashes)
{
if (!count($hashes)) {
return array();
//empty parameter
}
$dbr = $this->getSlaveDB();
$res = $dbr->select('image', LocalFile::selectFields(), array('img_sha1' => $hashes), __METHOD__, array('ORDER BY' => 'img_name'));
$result = array();
foreach ($res as $row) {
$file = $this->newFileFromRow($row);
$result[$file->getSha1()][] = $file;
}
$res->free();
return $result;
}
/**
* @param $resultPageSet ApiPageSet
* @return void
*/
private function run($resultPageSet = null)
{
$repo = $this->mRepo;
if (!$repo instanceof LocalRepo) {
$this->dieUsage('Local file repository does not support querying all images', 'unsupportedrepo');
}
$db = $this->getDB();
$params = $this->extractRequestParams();
if (!is_null($params['continue'])) {
$cont = explode('|', $params['continue']);
if (count($cont) != 1) {
$this->dieUsage("Invalid continue param. You should pass the " . "original value returned by the previous query", "_badcontinue");
}
$op = $params['dir'] == 'descending' ? '<' : '>';
$cont_from = $db->addQuotes($cont[0]);
$this->addWhere("img_name {$op}= {$cont_from}");
}
// Image filters
$dir = $params['dir'] == 'descending' ? 'older' : 'newer';
$from = is_null($params['from']) ? null : $this->titlePartToKey($params['from']);
$to = is_null($params['to']) ? null : $this->titlePartToKey($params['to']);
$this->addWhereRange('img_name', $dir, $from, $to);
if (isset($params['prefix'])) {
$this->addWhere('img_name' . $db->buildLike($this->titlePartToKey($params['prefix']), $db->anyString()));
}
if (isset($params['minsize'])) {
$this->addWhere('img_size>=' . intval($params['minsize']));
}
if (isset($params['maxsize'])) {
$this->addWhere('img_size<=' . intval($params['maxsize']));
}
$sha1 = false;
if (isset($params['sha1'])) {
if (!$this->validateSha1Hash($params['sha1'])) {
$this->dieUsage('The SHA1 hash provided is not valid', 'invalidsha1hash');
}
$sha1 = wfBaseConvert($params['sha1'], 16, 36, 31);
} elseif (isset($params['sha1base36'])) {
$sha1 = $params['sha1base36'];
if (!$this->validateSha1Base36Hash($sha1)) {
$this->dieUsage('The SHA1Base36 hash provided is not valid', 'invalidsha1base36hash');
}
}
if ($sha1) {
$this->addWhereFld('img_sha1', $sha1);
}
if (!is_null($params['mime'])) {
global $wgMiserMode;
if ($wgMiserMode) {
$this->dieUsage('MIME search disabled in Miser Mode', 'mimesearchdisabled');
}
list($major, $minor) = File::splitMime($params['mime']);
$this->addWhereFld('img_major_mime', $major);
$this->addWhereFld('img_minor_mime', $minor);
}
$this->addTables('image');
$prop = array_flip($params['prop']);
$this->addFields(LocalFile::selectFields());
$limit = $params['limit'];
$this->addOption('LIMIT', $limit + 1);
$sort = $params['dir'] == 'descending' ? ' DESC' : '';
$this->addOption('ORDER BY', 'img_name' . $sort);
$res = $this->select(__METHOD__);
$titles = array();
$count = 0;
$result = $this->getResult();
foreach ($res as $row) {
if (++$count > $limit) {
// We've reached the one extra which shows that there are additional pages to be had. Stop here...
$this->setContinueEnumParameter('continue', $row->img_name);
break;
}
if (is_null($resultPageSet)) {
$file = $repo->newFileFromRow($row);
$info = array_merge(array('name' => $row->img_name), ApiQueryImageInfo::getInfo($file, $prop, $result));
self::addTitleInfo($info, $file->getTitle());
$fit = $result->addValue(array('query', $this->getModuleName()), null, $info);
if (!$fit) {
$this->setContinueEnumParameter('continue', $row->img_name);
break;
}
} else {
$titles[] = Title::makeTitle(NS_FILE, $row->img_name);
}
}
if (is_null($resultPageSet)) {
$result->setIndexedTagName_internal(array('query', $this->getModuleName()), 'img');
} else {
$resultPageSet->populateFromTitles($titles);
}
}
private function run($resultPageSet = null)
{
$repo = RepoGroup::singleton()->getLocalRepo();
if (!$repo instanceof LocalRepo) {
$this->dieUsage('Local file repository does not support querying all images', 'unsupportedrepo');
}
$db = $this->getDB();
$params = $this->extractRequestParams();
// Image filters
$dir = $params['dir'] == 'descending' ? 'older' : 'newer';
$from = is_null($params['from']) ? null : $this->titlePartToKey($params['from']);
$this->addWhereRange('img_name', $dir, $from, null);
if (isset($params['prefix'])) {
$this->addWhere("img_name LIKE '" . $db->escapeLike($this->titlePartToKey($params['prefix'])) . "%'");
}
if (isset($params['minsize'])) {
$this->addWhere('img_size>=' . intval($params['minsize']));
}
if (isset($params['maxsize'])) {
$this->addWhere('img_size<=' . intval($params['maxsize']));
}
$sha1 = false;
if (isset($params['sha1'])) {
$sha1 = wfBaseConvert($params['sha1'], 16, 36, 31);
} elseif (isset($params['sha1base36'])) {
$sha1 = $params['sha1base36'];
}
if ($sha1) {
$this->addWhere('img_sha1=' . $db->addQuotes($sha1));
}
$this->addTables('image');
$prop = array_flip($params['prop']);
$this->addFields(LocalFile::selectFields());
$limit = $params['limit'];
$this->addOption('LIMIT', $limit + 1);
$this->addOption('ORDER BY', 'img_name' . ($params['dir'] == 'descending' ? ' DESC' : ''));
$res = $this->select(__METHOD__);
$titles = array();
$count = 0;
$result = $this->getResult();
while ($row = $db->fetchObject($res)) {
if (++$count > $limit) {
// We've reached the one extra which shows that there are additional pages to be had. Stop here...
// TODO: Security issue - if the user has no right to view next title, it will still be shown
$this->setContinueEnumParameter('from', $this->keyToTitle($row->img_name));
break;
}
if (is_null($resultPageSet)) {
$file = $repo->newFileFromRow($row);
$info = array_merge(array('name' => $row->img_name), ApiQueryImageInfo::getInfo($file, $prop, $result));
$fit = $result->addValue(array('query', $this->getModuleName()), null, $info);
if (!$fit) {
$this->setContinueEnumParameter('from', $this->keyToTitle($row->img_name));
break;
}
} else {
$titles[] = Title::makeTitle(NS_IMAGE, $row->img_name);
}
}
$db->freeResult($res);
if (is_null($resultPageSet)) {
$result->setIndexedTagName_internal(array('query', $this->getModuleName()), 'img');
} else {
$resultPageSet->populateFromTitles($titles);
}
}
public function execute()
{
global $wgUser;
$wgUser = User::newFromName(self::USER);
$this->isDryRun = $this->hasOption('dry-run');
$this->otherLocation = $this->getOption('other-location');
$this->repo = RepoGroup::singleton()->getLocalRepo();
$this->dbr = $this->getDB(DB_SLAVE);
$images = 0;
$imagesMissing = 0;
$imagesFixed = 0;
$res = $this->dbr->select('image', LocalFile::selectFields(), '', __METHOD__);
$count = $res->numRows();
$this->output(sprintf("Checking all images (%d images)%s...\n\n", $count, $this->isDryRun ? ' in dry run mode' : ''));
while ($row = $res->fetchObject()) {
$file = LocalFile::newFromRow($row, $this->repo);
// 1. check file size (img_size = 0 in image table)
$this->checkFileSize($file);
// 2. check for missing files on FS / Swift storage
$result = $this->processMissingFile($file);
switch ($result) {
case self::RESULT_RESTORED:
$imagesFixed++;
$imagesMissing++;
break;
case self::RESULT_NOT_RESTORED:
$imagesMissing++;
break;
}
// progress
if (++$images % 100) {
$this->output(sprintf("%d%%\r", $images / $count * 100));
}
}
// summary
if (!empty($this->otherLocation)) {
$this->output(sprintf("Restored %d images from second location \n", count($this->foundMissing)));
$this->output("List of restored files: \n");
$this->output(sprintf("%s\n\n", implode("\t\n", $this->foundMissing)));
}
$this->output(sprintf("Detected %d missing images (%.2f%% of %d images) and fixed %d images\n\n", $imagesMissing, $imagesMissing / $count * 100, $count, $imagesFixed));
}
/**
* @param $resultPageSet ApiPageSet
* @return void
*/
private function run($resultPageSet = null)
{
$repo = $this->mRepo;
if (!$repo instanceof LocalRepo) {
$this->dieUsage('Local file repository does not support querying all images', 'unsupportedrepo');
}
$prefix = $this->getModulePrefix();
$db = $this->getDB();
$params = $this->extractRequestParams();
// Table and return fields
$this->addTables('image');
$prop = array_flip($params['prop']);
$this->addFields(LocalFile::selectFields());
$dir = in_array($params['dir'], array('descending', 'older')) ? 'older' : 'newer';
if ($params['sort'] == 'name') {
// Check mutually exclusive params
$disallowed = array('start', 'end', 'user');
foreach ($disallowed as $pname) {
if (isset($params[$pname])) {
$this->dieUsage("Parameter '{$prefix}{$pname}' can only be used with {$prefix}sort=timestamp", 'badparams');
}
}
if ($params['filterbots'] != 'all') {
$this->dieUsage("Parameter '{$prefix}filterbots' can only be used with {$prefix}sort=timestamp", 'badparams');
}
// Pagination
if (!is_null($params['continue'])) {
$cont = explode('|', $params['continue']);
if (count($cont) != 1) {
$this->dieUsage('Invalid continue param. You should pass the ' . 'original value returned by the previous query', '_badcontinue');
}
$op = $dir == 'older' ? '<' : '>';
$cont_from = $db->addQuotes($cont[0]);
$this->addWhere("img_name {$op}= {$cont_from}");
}
// Image filters
$from = is_null($params['from']) ? null : $this->titlePartToKey($params['from']);
$to = is_null($params['to']) ? null : $this->titlePartToKey($params['to']);
$this->addWhereRange('img_name', $dir, $from, $to);
if (isset($params['prefix'])) {
$this->addWhere('img_name' . $db->buildLike($this->titlePartToKey($params['prefix']), $db->anyString()));
}
} else {
// Check mutually exclusive params
$disallowed = array('from', 'to', 'prefix');
foreach ($disallowed as $pname) {
if (isset($params[$pname])) {
$this->dieUsage("Parameter '{$prefix}{$pname}' can only be used with {$prefix}sort=name", 'badparams');
}
}
if (!is_null($params['user']) && $params['filterbots'] != 'all') {
// Since filterbots checks if each user has the bot right, it doesn't make sense to use it with user
$this->dieUsage("Parameters 'user' and 'filterbots' cannot be used together", 'badparams');
}
// Pagination
$this->addTimestampWhereRange('img_timestamp', $dir, $params['start'], $params['end']);
// Image filters
if (!is_null($params['user'])) {
$this->addWhereFld('img_user_text', $params['user']);
}
if ($params['filterbots'] != 'all') {
$this->addTables('user_groups');
$groupCond = $params['filterbots'] == 'nobots' ? 'NULL' : 'NOT NULL';
$this->addWhere("ug_group IS {$groupCond}");
$this->addJoinConds(array('user_groups' => array('LEFT JOIN', array('ug_group' => User::getGroupsWithPermission('bot'), 'ug_user = img_user'))));
}
}
// Filters not depending on sort
if (isset($params['minsize'])) {
$this->addWhere('img_size>=' . intval($params['minsize']));
}
if (isset($params['maxsize'])) {
$this->addWhere('img_size<=' . intval($params['maxsize']));
}
$sha1 = false;
if (isset($params['sha1'])) {
if (!$this->validateSha1Hash($params['sha1'])) {
$this->dieUsage('The SHA1 hash provided is not valid', 'invalidsha1hash');
}
$sha1 = wfBaseConvert($params['sha1'], 16, 36, 31);
} elseif (isset($params['sha1base36'])) {
$sha1 = $params['sha1base36'];
if (!$this->validateSha1Base36Hash($sha1)) {
$this->dieUsage('The SHA1Base36 hash provided is not valid', 'invalidsha1base36hash');
}
}
if ($sha1) {
$this->addWhereFld('img_sha1', $sha1);
}
if (!is_null($params['mime'])) {
global $wgMiserMode;
if ($wgMiserMode) {
$this->dieUsage('MIME search disabled in Miser Mode', 'mimesearchdisabled');
}
list($major, $minor) = File::splitMime($params['mime']);
$this->addWhereFld('img_major_mime', $major);
$this->addWhereFld('img_minor_mime', $minor);
}
$limit = $params['limit'];
$this->addOption('LIMIT', $limit + 1);
$sort = $dir == 'older' ? ' DESC' : '';
if ($params['sort'] == 'timestamp') {
$this->addOption('ORDER BY', 'img_timestamp' . $sort);
if ($params['filterbots'] == 'all') {
$this->addOption('USE INDEX', array('image' => 'img_timestamp'));
} else {
$this->addOption('USE INDEX', array('image' => 'img_usertext_timestamp'));
}
} else {
$this->addOption('ORDER BY', 'img_name' . $sort);
}
$res = $this->select(__METHOD__);
$titles = array();
$count = 0;
$result = $this->getResult();
foreach ($res as $row) {
if (++$count > $limit) {
// We've reached the one extra which shows that there are additional pages to be had. Stop here...
if ($params['sort'] == 'name') {
$this->setContinueEnumParameter('continue', $row->img_name);
} else {
$this->setContinueEnumParameter('start', wfTimestamp(TS_ISO_8601, $row->img_timestamp));
}
break;
}
if (is_null($resultPageSet)) {
$file = $repo->newFileFromRow($row);
$info = array_merge(array('name' => $row->img_name), ApiQueryImageInfo::getInfo($file, $prop, $result));
self::addTitleInfo($info, $file->getTitle());
$fit = $result->addValue(array('query', $this->getModuleName()), null, $info);
if (!$fit) {
if ($params['sort'] == 'name') {
$this->setContinueEnumParameter('continue', $row->img_name);
} else {
$this->setContinueEnumParameter('start', wfTimestamp(TS_ISO_8601, $row->img_timestamp));
}
break;
}
} else {
$titles[] = Title::makeTitle(NS_FILE, $row->img_name);
}
}
if (is_null($resultPageSet)) {
$result->setIndexedTagName_internal(array('query', $this->getModuleName()), 'img');
} else {
$resultPageSet->populateFromTitles($titles);
}
}
