/**
* Helper function -- given a file on the filesystem, find matching content in the db (and associated articles) and remove them.
* @param $filePath String: path to file on the filesystem
*/
public function deleteFileByContent($filePath)
{
$hash = File::sha1Base36($filePath);
$dupes = RepoGroup::singleton()->findBySha1($hash);
$success = true;
foreach ($dupes as $dupe) {
$success &= $this->deleteFileByTitle($dupe->getTitle());
}
return $success;
}
public function execute()
{
$method = $this->getOption('method', 'normal');
$file = $this->getOption('file');
$t = -microtime(true);
$dbw = wfGetDB(DB_MASTER);
if ($file) {
$res = $dbw->selectRow('image', array('img_name'), array('img_name' => $dbw->addQuotes($file)), __METHOD__);
if (!$res) {
$this->error("No such file: {$file}", true);
return;
}
} else {
$res = $dbw->select('image', array('img_name'), array('img_sha1' => ''), __METHOD__);
}
$imageTable = $dbw->tableName('image');
$oldimageTable = $dbw->tableName('oldimage');
$batch = array();
if ($method == 'pipe') {
// @fixme kill this and replace with a second unbuffered DB connection.
global $wgDBuser, $wgDBserver, $wgDBpassword, $wgDBname;
$cmd = 'mysql -u' . wfEscapeShellArg($wgDBuser) . ' -h' . wfEscapeShellArg($wgDBserver) . ' -p' . wfEscapeShellArg($wgDBpassword, $wgDBname);
$this->output("Using pipe method\n");
$pipe = popen($cmd, 'w');
}
$numRows = $res->numRows();
$i = 0;
foreach ($res as $row) {
if ($i % 100 == 0) {
$this->output(sprintf("Done %d of %d, %5.3f%% \r", $i, $numRows, $i / $numRows * 100));
wfWaitForSlaves(5);
}
$file = wfLocalFile($row->img_name);
if (!$file) {
continue;
}
$sha1 = File::sha1Base36($file->getPath());
if (strval($sha1) !== '') {
$sql = "UPDATE {$imageTable} SET img_sha1=" . $dbw->addQuotes($sha1) . " WHERE img_name=" . $dbw->addQuotes($row->img_name);
if ($method == 'pipe') {
fwrite($pipe, "{$sql};\n");
} else {
$dbw->query($sql, __METHOD__);
}
}
$i++;
}
if ($method == 'pipe') {
fflush($pipe);
pclose($pipe);
}
$t += microtime(true);
$this->output(sprintf("\nDone %d files in %.1f seconds\n", $numRows, $t));
}
}
# Check existence
$image = wfLocalFile($title);
if ($image->exists()) {
if (isset($options['overwrite'])) {
echo "{$base} exists, overwriting...";
$svar = 'overwritten';
} else {
echo "{$base} exists, skipping\n";
$skipped++;
continue;
}
} else {
if (isset($options['skip-dupes'])) {
$repo = $image->getRepo();
$sha1 = File::sha1Base36($file);
# XXX: we end up calculating this again when actually uploading. that sucks.
$dupes = $repo->findBySha1($sha1);
if ($dupes) {
echo "{$base} already exists as " . $dupes[0]->getName() . ", skipping\n";
$skipped++;
continue;
}
}
echo "Importing {$base}...";
$svar = 'added';
}
if (isset($options['source-wiki-url'])) {
/* find comment text directly from source wiki, through MW's API */
$real_comment = getFileCommentFromSourceWiki($options['source-wiki-url'], $base);
if ($real_comment === false) {
/**
* Move or copy a file to a specified location. Returns a FileRepoStatus
* object with the archive name in the "value" member on success.
*
* The archive name should be passed through to recordUpload for database
* registration.
*
* @param string $srcPath Local filesystem path or virtual URL to the source image
* @param string $dstRel Target relative path
* @param int $flags A bitwise combination of:
* File::DELETE_SOURCE Delete the source file, i.e. move rather than copy
* @param array $options Optional additional parameters
* @return FileRepoStatus On success, the value member contains the
* archive name, or an empty string if it was a new file.
*/
function publishTo($srcPath, $dstRel, $flags = 0, array $options = array())
{
$repo = $this->getRepo();
if ($repo->getReadOnlyReason() !== false) {
return $this->readOnlyFatalStatus();
}
$this->lock();
// begin
$archiveName = wfTimestamp(TS_MW) . '!' . $this->getName();
$archiveRel = 'archive/' . $this->getHashPath() . $archiveName;
if ($repo->hasSha1Storage()) {
$sha1 = $repo->isVirtualUrl($srcPath) ? $repo->getFileSha1($srcPath) : File::sha1Base36($srcPath);
$dst = $repo->getBackend()->getPathForSHA1($sha1);
$status = $repo->quickImport($srcPath, $dst);
if ($flags & File::DELETE_SOURCE) {
unlink($srcPath);
}
if ($this->exists()) {
$status->value = $archiveName;
}
} else {
$flags = $flags & File::DELETE_SOURCE ? LocalRepo::DELETE_SOURCE : 0;
$status = $repo->publish($srcPath, $dstRel, $archiveRel, $flags, $options);
if ($status->value == 'new') {
$status->value = '';
} else {
$status->value = $archiveName;
}
}
$this->unlock();
// done
return $status;
}
function getSha1()
{
$this->load();
// Initialise now if necessary
if ($this->sha1 == '' && $this->fileExists) {
$this->sha1 = File::sha1Base36($this->getPath());
if (!wfReadOnly() && strval($this->sha1) != '') {
$dbw = $this->repo->getMasterDB();
$dbw->update('image', array('img_sha1' => $this->sha1), array('img_name' => $this->getName()), __METHOD__);
$this->saveToCache();
}
}
return $this->sha1;
}
/**
* Check for non fatal problems with the file
*
* @return Array of warnings
*/
public function checkWarnings()
{
$warnings = array();
$localFile = $this->getLocalFile();
$filename = $localFile->getName();
$n = strrpos($filename, '.');
/**
* Check whether the resulting filename is different from the desired one,
* but ignore things like ucfirst() and spaces/underscore things
*/
$comparableName = str_replace(' ', '_', $this->mDesiredDestName);
$comparableName = Title::capitalize($comparableName, NS_FILE);
if ($this->mDesiredDestName != $filename && $comparableName != $filename) {
$warnings['badfilename'] = $filename;
}
// Check whether the file extension is on the unwanted list
global $wgCheckFileExtensions, $wgFileExtensions;
if ($wgCheckFileExtensions) {
if (!$this->checkFileExtension($this->mFinalExtension, $wgFileExtensions)) {
$warnings['filetype-unwanted-type'] = $this->mFinalExtension;
}
}
global $wgUploadSizeWarning;
if ($wgUploadSizeWarning && $this->mFileSize > $wgUploadSizeWarning) {
$warnings['large-file'] = $wgUploadSizeWarning;
}
if ($this->mFileSize == 0) {
$warnings['emptyfile'] = true;
}
$exists = self::getExistsWarning($localFile);
if ($exists !== false) {
$warnings['exists'] = $exists;
}
// Check dupes against existing files
$hash = File::sha1Base36($this->mTempPath);
$dupes = RepoGroup::singleton()->findBySha1($hash);
$title = $this->getTitle();
// Remove all matches against self
foreach ($dupes as $key => $dupe) {
if ($title->equals($dupe->getTitle())) {
unset($dupes[$key]);
}
}
if ($dupes) {
$warnings['duplicate'] = $dupes;
}
// Check dupes against archives
$archivedImage = new ArchivedFile(null, 0, "{$hash}.{$this->mFinalExtension}");
if ($archivedImage->getID() > 0) {
$warnings['duplicate-archive'] = $archivedImage->getName();
}
return $warnings;
}
/**
* Check for duplicate files and throw up a warning before the upload
* completes.
*/
function getDupeWarning($tempfile, $extension, $destinationTitle)
{
$hash = File::sha1Base36($tempfile);
$dupes = RepoGroup::singleton()->findBySha1($hash);
$archivedImage = new ArchivedFile(null, 0, $hash . ".{$extension}");
if ($dupes) {
global $wgOut;
$msg = "<gallery>";
foreach ($dupes as $file) {
$title = $file->getTitle();
# Don't throw the warning when the titles are the same, it's a reupload
# and highly redundant.
if (!$title->equals($destinationTitle) || !$this->mForReUpload) {
$msg .= $title->getPrefixedText() . "|" . $title->getText() . "\n";
}
}
$msg .= "</gallery>";
return "<li>" . wfMsgExt("file-exists-duplicate", array("parse"), count($dupes)) . $wgOut->parse($msg) . "</li>\n";
} elseif ($archivedImage->getID() > 0) {
global $wgOut;
$name = Title::makeTitle(NS_FILE, $archivedImage->getName())->getPrefixedText();
return Xml::tags('li', null, wfMsgExt('file-deleted-duplicate', array('parseinline'), array($name)));
} else {
return '';
}
}
if ($method == 'pipe') {
echo "Using pipe method\n";
$pipe = popen($cmd, 'w');
}
$numRows = $res->numRows();
$i = 0;
foreach ($res as $row) {
if ($i % 100 == 0) {
printf("Done %d of %d, %5.3f%% \r", $i, $numRows, $i / $numRows * 100);
wfWaitForSlaves(5);
}
$file = wfLocalFile($row->img_name);
if (!$file) {
continue;
}
$sha1 = File::sha1Base36($file->getPath());
if (strval($sha1) !== '') {
$sql = "UPDATE {$imageTable} SET img_sha1=" . $dbw->addQuotes($sha1) . " WHERE img_name=" . $dbw->addQuotes($row->img_name);
if ($method == 'pipe') {
fwrite($pipe, "{$sql};\n");
} else {
$dbw->query($sql, $fname);
}
}
$i++;
}
if ($method == 'pipe') {
fflush($pipe);
pclose($pipe);
}
$t += microtime(true);
<?php
require_once dirname(__FILE__) . '/../commandLine.inc';
$dbr = wfGetDB(DB_SLAVE);
$dbw = wfGetDB(DB_MASTER);
$sth = $dbr->select(array("image"), array("img_name", "img_media_type"), false, __METHOD__);
while ($row = $dbr->fetchObject($sth)) {
$file = wfLocalFile($row->img_name);
if ($file) {
$oldFiles = $file->getHistory();
foreach ($oldFiles as $oldfile) {
$path = $oldfile->getPath();
$parts = explode("/", $path);
$file = array_pop($parts);
if (is_file($path)) {
$sha1 = File::sha1Base36($path);
/**
* select row from old image
*/
$row = $dbr->selectRow(array("oldimage"), array("*"), array("oi_archive_name" => $file), __METHOD__);
if ($row->oi_sha1 !== $sha1) {
Wikia::log("info", "", "{$path} new:{$sha1} <> old:{$row->oi_sha1}");
$dbw->update("oldimage", array("oi_sha1" => $sha1), array("oi_archive_name" => $file), __METHOD__);
}
} else {
Wikia::log("err", "", "{$path} {$file} doesn't exists");
}
}
}
}
$dbr->freeResult($sth);
