*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
* @ingroup Maintenance ExternalStorage
*/
$optionsWithArgs = RecompressTracked::getOptionsWithArgs();
require __DIR__ . '/../commandLine.inc';
if (count($args) < 1) {
echo "Usage: php recompressTracked.php [options] <cluster> [... <cluster>...]\nMoves blobs indexed by trackBlobs.php to a specified list of destination clusters, and recompresses them in the process. Restartable.\n\nOptions:\n\t--procs <procs> Set the number of child processes (default 1)\n\t--copy-only Copy only, do not update the text table. Restart without this option to complete.\n\t--debug-log <file> Log debugging data to the specified file\n\t--info-log <file> Log progress messages to the specified file\n\t--critical-log <file> Log error messages to the specified file\n";
exit(1);
}
$job = RecompressTracked::newFromCommandLine($args, $options);
$job->execute();
/**
* Maintenance script that moves blobs indexed by trackBlobs.php to a specified
* list of destination clusters, and recompresses them in the process.
*
* @ingroup Maintenance ExternalStorage
*/
class RecompressTracked
{
public $destClusters;
public $batchSize = 1000;
public $orphanBatchSize = 1000;
public $reportingInterval = 10;
public $numProcs = 1;
public $useDiff, $pageBlobClass, $orphanBlobClass;
/**
* Commit the blob.
* Does nothing if no text items have been added.
* May skip the move if --copy-only is set.
*/
function commit()
{
$originalCount = count($this->texts);
if (!$originalCount) {
return;
}
/* Check to see if the target text_ids have been moved already.
*
* We originally read from the slave, so this can happen when a single
* text_id is shared between multiple pages. It's rare, but possible
* if a delete/move/undelete cycle splits up a null edit.
*
* We do a locking read to prevent closer-run race conditions.
*/
$dbw = wfGetDB(DB_MASTER);
$dbw->begin(__METHOD__);
$res = $dbw->select('blob_tracking', ['bt_text_id', 'bt_moved'], ['bt_text_id' => array_keys($this->referrers)], __METHOD__, ['FOR UPDATE']);
$dirty = false;
foreach ($res as $row) {
if ($row->bt_moved) {
# This row has already been moved, remove it
$this->parent->debug("TRX: conflict detected in old_id={$row->bt_text_id}");
unset($this->texts[$row->bt_text_id]);
$dirty = true;
}
}
// Recompress the blob if necessary
if ($dirty) {
if (!count($this->texts)) {
// All have been moved already
if ($originalCount > 1) {
// This is suspcious, make noise
$this->parent->critical("Warning: concurrent operation detected, are there two conflicting " . "processes running, doing the same job?");
}
return;
}
$this->recompress();
}
// Insert the data into the destination cluster
$targetCluster = $this->parent->getTargetCluster();
$store = $this->parent->store;
$targetDB = $store->getMaster($targetCluster);
$targetDB->clearFlag(DBO_TRX);
// we manage the transactions
$targetDB->begin(__METHOD__);
$baseUrl = $this->parent->store->store($targetCluster, serialize($this->cgz));
// Write the new URLs to the blob_tracking table
foreach ($this->referrers as $textId => $hash) {
$url = $baseUrl . '/' . $hash;
$dbw->update('blob_tracking', ['bt_new_url' => $url], ['bt_text_id' => $textId, 'bt_moved' => 0], __METHOD__);
}
$targetDB->commit(__METHOD__);
// Critical section here: interruption at this point causes blob duplication
// Reversing the order of the commits would cause data loss instead
$dbw->commit(__METHOD__);
// Write the new URLs to the text table and set the moved flag
if (!$this->parent->copyOnly) {
foreach ($this->referrers as $textId => $hash) {
$url = $baseUrl . '/' . $hash;
$this->parent->moveTextRow($textId, $url);
}
}
}
