private function loadGitCommitRef()
{
$repository = $this->getRepository();
// NOTE: %B was introduced somewhat recently in git's history, so pull
// commit message information with %s and %b instead.
// Even though we pass --encoding here, git doesn't always succeed, so
// we try a little harder, since git *does* tell us what the actual encoding
// is correctly (unless it doesn't; encoding is sometimes empty).
list($info) = $repository->execxLocalCommand('log -n 1 --encoding=%s --format=%s %s --', 'UTF-8', implode('%x00', array('%e', '%cn', '%ce', '%an', '%ae', '%T', '%at', '%s%n%n%b')), $this->identifier);
$parts = explode("", $info);
$encoding = array_shift($parts);
foreach ($parts as $key => $part) {
if ($encoding) {
$part = phutil_utf8_convert($part, 'UTF-8', $encoding);
}
$parts[$key] = phutil_utf8ize($part);
if (!strlen($parts[$key])) {
$parts[$key] = null;
}
}
$hashes = array(id(new DiffusionCommitHash())->setHashType(ArcanistDifferentialRevisionHash::HASH_GIT_COMMIT)->setHashValue($this->identifier), id(new DiffusionCommitHash())->setHashType(ArcanistDifferentialRevisionHash::HASH_GIT_TREE)->setHashValue($parts[4]));
$author_epoch = (int) $parts[5];
if (!$author_epoch) {
$author_epoch = null;
}
return id(new DiffusionCommitRef())->setCommitterName($parts[0])->setCommitterEmail($parts[1])->setAuthorName($parts[2])->setAuthorEmail($parts[3])->setHashes($hashes)->setAuthorEpoch($author_epoch)->setMessage($parts[6]);
}
public final function loadFileContentFromFuture(Future $future)
{
if ($this->timeout) {
$future->setTimeout($this->timeout);
}
if ($this->getByteLimit()) {
$future->setStdoutSizeLimit($this->getByteLimit());
}
try {
$file_content = $this->executeQueryFromFuture($future);
} catch (CommandException $ex) {
if (!$future->getWasKilledByTimeout()) {
throw $ex;
}
$message = pht('<Attempt to load this file was terminated after %s second(s).>', $this->timeout);
$file_content = new DiffusionFileContent();
$file_content->setCorpus($message);
}
$this->fileContent = $file_content;
$repository = $this->getRequest()->getRepository();
$try_encoding = $repository->getDetail('encoding');
if ($try_encoding) {
$this->fileContent->setCorpus(phutil_utf8_convert($this->fileContent->getCorpus(), 'UTF-8', $try_encoding));
}
return $this->fileContent;
}
public final function loadFileContent()
{
$this->fileContent = $this->executeQuery();
$repository = $this->getRequest()->getRepository();
$try_encoding = $repository->getDetail('encoding');
if ($try_encoding) {
$this->fileContent->setCorpus(phutil_utf8_convert($this->fileContent->getCorpus(), "UTF-8", $try_encoding));
}
return $this->fileContent;
}
public function parseCommit(PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit)
{
// NOTE: %B was introduced somewhat recently in git's history, so pull
// commit message information with %s and %b instead.
// Even though we pass --encoding here, git doesn't always succeed, so
// we try a little harder, since git *does* tell us what the actual encoding
// is correctly (unless it doesn't; encoding is sometimes empty).
list($info) = $repository->execxLocalCommand('log -n 1 --encoding=%s --format=%s %s --', 'UTF-8', implode('%x00', array('%e', '%cn', '%ce', '%an', '%ae', '%s%n%n%b')), $commit->getCommitIdentifier());
$parts = explode("", $info);
$encoding = array_shift($parts);
foreach ($parts as $key => $part) {
if ($encoding) {
$part = phutil_utf8_convert($part, 'UTF-8', $encoding);
}
$parts[$key] = phutil_utf8ize($part);
}
$committer_name = $parts[0];
$committer_email = $parts[1];
$author_name = $parts[2];
$author_email = $parts[3];
$message = $parts[4];
if (strlen($author_email)) {
$author = "{$author_name} <{$author_email}>";
} else {
$author = "{$author_name}";
}
if (strlen($committer_email)) {
$committer = "{$committer_name} <{$committer_email}>";
} else {
$committer = "{$committer_name}";
}
if ($committer == $author) {
$committer = null;
}
$this->updateCommitData($author, $message, $committer);
if ($this->shouldQueueFollowupTasks()) {
$task = new PhabricatorWorkerTask();
$task->setTaskClass('PhabricatorRepositoryGitCommitChangeParserWorker');
$task->setData(array('commitID' => $commit->getID()));
$task->save();
}
}
public function __construct($status_code, $body, array $headers, $expect = null)
{
// NOTE: Avoiding PhutilUTF8StringTruncator here because this isn't lazy
// and responses may be large.
if (strlen($body) > 512) {
$excerpt = substr($body, 0, 512) . '...';
} else {
$excerpt = $body;
}
$content_type = BaseHTTPFuture::getHeader($headers, 'Content-Type');
$match = null;
if (preg_match('/;\\s*charset=([^;]+)/', $content_type, $match)) {
$encoding = trim($match[1], "\"'");
try {
$excerpt = phutil_utf8_convert($excerpt, 'UTF-8', $encoding);
} catch (Exception $ex) {
}
}
$this->excerpt = phutil_utf8ize($excerpt);
$this->expect = $expect;
parent::__construct($status_code);
}
* See the License for the specific language governing permissions and
* limitations under the License.
*/
if ($argc > 1) {
$_SERVER['PHABRICATOR_ENV'] = $argv[1];
}
$root = dirname(dirname(dirname(__FILE__)));
require_once $root . '/scripts/__init_script__.php';
require_once $root . '/externals/mimemailparser/MimeMailParser.class.php';
$parser = new MimeMailParser();
$parser->setText(file_get_contents('php://stdin'));
$text_body = $parser->getMessageBody('text');
$text_body_headers = $parser->getMessageBodyHeaders('text');
$content_type = idx($text_body_headers, 'content-type');
if (!phutil_is_utf8($text_body) && (preg_match('/charset="(.*?)"/', $content_type, $matches) || preg_match('/charset=(\\S+)/', $content_type, $matches))) {
$text_body = phutil_utf8_convert($text_body, "UTF-8", $matches[1]);
}
$headers = $parser->getHeaders();
$headers['subject'] = iconv_mime_decode($headers['subject'], 0, "UTF-8");
$headers['from'] = iconv_mime_decode($headers['from'], 0, "UTF-8");
$received = new PhabricatorMetaMTAReceivedMail();
$received->setHeaders($headers);
$received->setBodies(array('text' => $text_body, 'html' => $parser->getMessageBody('html')));
$attachments = array();
foreach ($parser->getAttachments() as $attachment) {
if (preg_match('@text/(plain|html)@', $attachment->getContentType()) && $attachment->getContentDisposition() == 'inline') {
// If this is an "inline" attachment with some sort of text content-type,
// do not treat it as a file for attachment. MimeMailParser already picked
// it up in the getMessageBody() call above. We still want to treat 'inline'
// attachments with other content types (e.g., images) as attachments.
continue;
protected function generateChanges()
{
$parser = $this->newDiffParser();
$is_raw = $this->isRawDiffSource();
if ($is_raw) {
if ($this->getArgument('raw')) {
fwrite(STDERR, pht('Reading diff from stdin...') . "\n");
$raw_diff = file_get_contents('php://stdin');
} else {
if ($this->getArgument('raw-command')) {
list($raw_diff) = execx('%C', $this->getArgument('raw-command'));
} else {
throw new Exception(pht('Unknown raw diff source.'));
}
}
$changes = $parser->parseDiff($raw_diff);
foreach ($changes as $key => $change) {
// Remove "message" changes, e.g. from "git show".
if ($change->getType() == ArcanistDiffChangeType::TYPE_MESSAGE) {
unset($changes[$key]);
}
}
return $changes;
}
$repository_api = $this->getRepositoryAPI();
if ($repository_api instanceof ArcanistSubversionAPI) {
$paths = $this->generateAffectedPaths();
$this->primeSubversionWorkingCopyData($paths);
// Check to make sure the user is diffing from a consistent base revision.
// This is mostly just an abuse sanity check because it's silly to do this
// and makes the code more difficult to effectively review, but it also
// affects patches and makes them nonportable.
$bases = $repository_api->getSVNBaseRevisions();
// Remove all files with baserev "0"; these files are new.
foreach ($bases as $path => $baserev) {
if ($bases[$path] <= 0) {
unset($bases[$path]);
}
}
if ($bases) {
$rev = reset($bases);
$revlist = array();
foreach ($bases as $path => $baserev) {
$revlist[] = ' ' . pht('Revision %s, %s', $baserev, $path);
}
$revlist = implode("\n", $revlist);
foreach ($bases as $path => $baserev) {
if ($baserev !== $rev) {
throw new ArcanistUsageException(pht("Base revisions of changed paths are mismatched. Update all " . "paths to the same base revision before creating a diff: " . "\n\n%s", $revlist));
}
}
// If you have a change which affects several files, all of which are
// at a consistent base revision, treat that revision as the effective
// base revision. The use case here is that you made a change to some
// file, which updates it to HEAD, but want to be able to change it
// again without updating the entire working copy. This is a little
// sketchy but it arises in Facebook Ops workflows with config files and
// doesn't have any real material tradeoffs (e.g., these patches are
// perfectly applyable).
$repository_api->overrideSVNBaseRevisionNumber($rev);
}
$changes = $parser->parseSubversionDiff($repository_api, $paths);
} else {
if ($repository_api instanceof ArcanistGitAPI) {
$diff = $repository_api->getFullGitDiff($repository_api->getBaseCommit(), $repository_api->getHeadCommit());
if (!strlen($diff)) {
throw new ArcanistUsageException(pht('No changes found. (Did you specify the wrong commit range?)'));
}
$changes = $parser->parseDiff($diff);
} else {
if ($repository_api instanceof ArcanistMercurialAPI) {
$diff = $repository_api->getFullMercurialDiff();
if (!strlen($diff)) {
throw new ArcanistUsageException(pht('No changes found. (Did you specify the wrong commit range?)'));
}
$changes = $parser->parseDiff($diff);
} else {
throw new Exception(pht('Repository API is not supported.'));
}
}
}
if (count($changes) > 250) {
$message = pht('This diff has a very large number of changes (%s). Differential ' . 'works best for changes which will receive detailed human review, ' . 'and not as well for large automated changes or bulk checkins. ' . 'See %s for information about reviewing big checkins. Continue anyway?', phutil_count($changes), 'https://secure.phabricator.com/book/phabricator/article/' . 'differential_large_changes/');
if (!phutil_console_confirm($message)) {
throw new ArcanistUsageException(pht('Aborted generation of gigantic diff.'));
}
}
$limit = 1024 * 1024 * 4;
foreach ($changes as $change) {
$size = 0;
foreach ($change->getHunks() as $hunk) {
$size += strlen($hunk->getCorpus());
}
if ($size > $limit) {
$byte_warning = pht("Diff for '%s' with context is %s bytes in length. " . "Generally, source changes should not be this large.", $change->getCurrentPath(), new PhutilNumber($size));
if (!$this->getArgument('less-context')) {
$byte_warning .= ' ' . pht("If this file is a huge text file, try using the '%s' flag.", '--less-context');
}
if ($repository_api instanceof ArcanistSubversionAPI) {
throw new ArcanistUsageException($byte_warning . ' ' . pht("If the file is not a text file, mark it as binary with:" . "\n\n \$ %s\n", 'svn propset svn:mime-type application/octet-stream <filename>'));
} else {
$confirm = $byte_warning . ' ' . pht("If the file is not a text file, you can mark it 'binary'. " . "Mark this file as 'binary' and continue?");
if (phutil_console_confirm($confirm)) {
$change->convertToBinaryChange($repository_api);
} else {
throw new ArcanistUsageException(pht('Aborted generation of gigantic diff.'));
}
}
}
}
$try_encoding = nonempty($this->getArgument('encoding'), null);
$utf8_problems = array();
foreach ($changes as $change) {
foreach ($change->getHunks() as $hunk) {
$corpus = $hunk->getCorpus();
if (!phutil_is_utf8($corpus)) {
// If this corpus is heuristically binary, don't try to convert it.
// mb_check_encoding() and mb_convert_encoding() are both very very
// liberal about what they're willing to process.
$is_binary = ArcanistDiffUtils::isHeuristicBinaryFile($corpus);
if (!$is_binary) {
if (!$try_encoding) {
try {
$try_encoding = $this->getRepositoryEncoding();
} catch (ConduitClientException $e) {
if ($e->getErrorCode() == 'ERR-BAD-ARCANIST-PROJECT') {
echo phutil_console_wrap(pht('Lookup of encoding in arcanist project failed: %s', $e->getMessage()) . "\n");
} else {
throw $e;
}
}
}
if ($try_encoding) {
$corpus = phutil_utf8_convert($corpus, 'UTF-8', $try_encoding);
$name = $change->getCurrentPath();
if (phutil_is_utf8($corpus)) {
$this->writeStatusMessage(pht("Converted a '%s' hunk from '%s' to UTF-8.\n", $name, $try_encoding));
$hunk->setCorpus($corpus);
continue;
}
}
}
$utf8_problems[] = $change;
break;
}
}
}
// If there are non-binary files which aren't valid UTF-8, warn the user
// and treat them as binary changes. See D327 for discussion of why Arcanist
// has this behavior.
if ($utf8_problems) {
$utf8_warning = sprintf("%s\n\n%s\n\n %s\n", pht('This diff includes %s file(s) which are not valid UTF-8 (they ' . 'contain invalid byte sequences). You can either stop this ' . 'workflow and fix these files, or continue. If you continue, ' . 'these files will be marked as binary.', phutil_count($utf8_problems)), pht("You can learn more about how Phabricator handles character " . "encodings (and how to configure encoding settings and detect and " . "correct encoding problems) by reading 'User Guide: UTF-8 and " . "Character Encoding' in the Phabricator documentation."), pht('%s AFFECTED FILE(S)', phutil_count($utf8_problems)));
$confirm = pht('Do you want to mark these %s file(s) as binary and continue?', phutil_count($utf8_problems));
echo phutil_console_format("**%s**\n", pht('Invalid Content Encoding (Non-UTF8)'));
echo phutil_console_wrap($utf8_warning);
$file_list = mpull($utf8_problems, 'getCurrentPath');
$file_list = ' ' . implode("\n ", $file_list);
echo $file_list;
if (!phutil_console_confirm($confirm, $default_no = false)) {
throw new ArcanistUsageException(pht('Aborted workflow to fix UTF-8.'));
} else {
foreach ($utf8_problems as $change) {
$change->convertToBinaryChange($repository_api);
}
}
}
$this->uploadFilesForChanges($changes);
return $changes;
}
public function testUTF8Convert()
{
if (!function_exists('mb_convert_encoding')) {
$this->assertSkipped("Requires mbstring extension.");
}
// "[ae]gis se[n]or [(c)] 1970 [+/-] 1 [degree]"
$input = "�gis SE�OR � 1970 �1�";
$expect = "ægis SEÑOR © 1970 ±1°";
$output = phutil_utf8_convert($input, 'UTF-8', 'ISO-8859-1');
$this->assertEqual($expect, $output, 'Conversion from ISO-8859-1.');
$caught = null;
try {
phutil_utf8_convert('xyz', 'moon language', 'UTF-8');
} catch (Exception $ex) {
$caught = $ex;
}
$this->assertEqual(true, (bool) $caught, 'Conversion with bogus encoding.');
}
protected function applyCustomInternalTransaction(PhabricatorLiskDAO $object, PhabricatorApplicationTransaction $xaction)
{
switch ($xaction->getTransactionType()) {
case PhabricatorRepositoryTransaction::TYPE_VCS:
$object->setVersionControlSystem($xaction->getNewValue());
break;
case PhabricatorRepositoryTransaction::TYPE_ACTIVATE:
$object->setDetail('tracking-enabled', $xaction->getNewValue());
break;
case PhabricatorRepositoryTransaction::TYPE_NAME:
$object->setName($xaction->getNewValue());
break;
case PhabricatorRepositoryTransaction::TYPE_DESCRIPTION:
$object->setDetail('description', $xaction->getNewValue());
break;
case PhabricatorRepositoryTransaction::TYPE_DEFAULT_BRANCH:
$object->setDetail('default-branch', $xaction->getNewValue());
break;
case PhabricatorRepositoryTransaction::TYPE_TRACK_ONLY:
$object->setDetail('branch-filter', array_fill_keys($xaction->getNewValue(), true));
break;
case PhabricatorRepositoryTransaction::TYPE_AUTOCLOSE_ONLY:
$object->setDetail('close-commits-filter', array_fill_keys($xaction->getNewValue(), true));
break;
case PhabricatorRepositoryTransaction::TYPE_UUID:
$object->setUUID($xaction->getNewValue());
break;
case PhabricatorRepositoryTransaction::TYPE_SVN_SUBPATH:
$object->setDetail('svn-subpath', $xaction->getNewValue());
break;
case PhabricatorRepositoryTransaction::TYPE_NOTIFY:
$object->setDetail('herald-disabled', (int) (!$xaction->getNewValue()));
break;
case PhabricatorRepositoryTransaction::TYPE_AUTOCLOSE:
$object->setDetail('disable-autoclose', (int) (!$xaction->getNewValue()));
break;
case PhabricatorRepositoryTransaction::TYPE_REMOTE_URI:
$object->setDetail('remote-uri', $xaction->getNewValue());
break;
case PhabricatorRepositoryTransaction::TYPE_LOCAL_PATH:
$object->setDetail('local-path', $xaction->getNewValue());
break;
case PhabricatorRepositoryTransaction::TYPE_HOSTING:
return $object->setHosted($xaction->getNewValue());
case PhabricatorRepositoryTransaction::TYPE_PROTOCOL_HTTP:
return $object->setServeOverHTTP($xaction->getNewValue());
case PhabricatorRepositoryTransaction::TYPE_PROTOCOL_SSH:
return $object->setServeOverSSH($xaction->getNewValue());
case PhabricatorRepositoryTransaction::TYPE_PUSH_POLICY:
return $object->setPushPolicy($xaction->getNewValue());
case PhabricatorRepositoryTransaction::TYPE_CREDENTIAL:
return $object->setCredentialPHID($xaction->getNewValue());
case PhabricatorRepositoryTransaction::TYPE_DANGEROUS:
$object->setDetail('allow-dangerous-changes', $xaction->getNewValue());
return;
case PhabricatorRepositoryTransaction::TYPE_SLUG:
$object->setRepositorySlug($xaction->getNewValue());
return;
case PhabricatorRepositoryTransaction::TYPE_SERVICE:
$object->setAlmanacServicePHID($xaction->getNewValue());
return;
case PhabricatorRepositoryTransaction::TYPE_SYMBOLS_LANGUAGE:
$object->setDetail('symbol-languages', $xaction->getNewValue());
return;
case PhabricatorRepositoryTransaction::TYPE_SYMBOLS_SOURCES:
$object->setDetail('symbol-sources', $xaction->getNewValue());
return;
case PhabricatorRepositoryTransaction::TYPE_STAGING_URI:
$object->setDetail('staging-uri', $xaction->getNewValue());
return;
case PhabricatorRepositoryTransaction::TYPE_AUTOMATION_BLUEPRINTS:
$object->setDetail('automation.blueprintPHIDs', $xaction->getNewValue());
return;
case PhabricatorRepositoryTransaction::TYPE_ENCODING:
// Make sure the encoding is valid by converting to UTF-8. This tests
// that the user has mbstring installed, and also that they didn't type
// a garbage encoding name. Note that we're converting from UTF-8 to
// the target encoding, because mbstring is fine with converting from
// a nonsense encoding.
$encoding = $xaction->getNewValue();
if (strlen($encoding)) {
try {
phutil_utf8_convert('.', $encoding, 'UTF-8');
} catch (Exception $ex) {
throw new PhutilProxyException(pht("Error setting repository encoding '%s': %s'", $encoding, $ex->getMessage()), $ex);
}
}
$object->setDetail('encoding', $encoding);
break;
}
}
protected function parseChangeset(ArcanistDiffChange $change)
{
// If a diff includes two sets of changes to the same file, let the
// second one win. In particular, this occurs when adding subdirectories
// in Subversion that contain files: the file text will be present in
// both the directory diff and the file diff. See T5555. Dropping the
// hunks lets whichever one shows up later win instead of showing changes
// twice.
$change->dropHunks();
$all_changes = array();
do {
$hunk = new ArcanistDiffHunk();
$line = $this->getLineTrimmed();
$real = array();
// In the case where only one line is changed, the length is omitted.
// The final group is for git, which appends a guess at the function
// context to the diff.
$matches = null;
$ok = preg_match('/^@@ -(\\d+)(?:,(\\d+))? \\+(\\d+)(?:,(\\d+))? @@(?: .*?)?$/U', $line, $matches);
if (!$ok) {
// It's possible we hit the style of an svn1.7 property change.
// This is a 4-line Index block, followed by an empty line, followed
// by a "Property changes on:" section similar to svn1.6.
if ($line == '') {
$line = $this->nextNonemptyLine();
$ok = preg_match('/^Property changes on:/', $line);
if (!$ok) {
$this->didFailParse(pht('Confused by empty line'));
}
$line = $this->nextLine();
return $this->parsePropertyHunk($change);
}
$this->didFailParse(pht("Expected hunk header '%s'.", '@@ -NN,NN +NN,NN @@'));
}
$hunk->setOldOffset($matches[1]);
$hunk->setNewOffset($matches[3]);
// Cover for the cases where length wasn't present (implying one line).
$old_len = idx($matches, 2);
if (!strlen($old_len)) {
$old_len = 1;
}
$new_len = idx($matches, 4);
if (!strlen($new_len)) {
$new_len = 1;
}
$hunk->setOldLength($old_len);
$hunk->setNewLength($new_len);
$add = 0;
$del = 0;
$hit_next_hunk = false;
while (($line = $this->nextLine()) !== null) {
if (strlen(rtrim($line, "\r\n"))) {
$char = $line[0];
} else {
// Normally, we do not encouter empty lines in diffs, because
// unchanged lines have an initial space. However, in Git, with
// the option `diff.suppress-blank-empty` set, unchanged blank lines
// emit as completely empty. If we encounter a completely empty line,
// treat it as a ' ' (i.e., unchanged empty line) line.
$char = ' ';
}
switch ($char) {
case '\\':
if (!preg_match('@\\ No newline at end of file@', $line)) {
$this->didFailParse(pht("Expected '\\ No newline at end of file'."));
}
if ($new_len) {
$real[] = $line;
$hunk->setIsMissingOldNewline(true);
} else {
$real[] = $line;
$hunk->setIsMissingNewNewline(true);
}
if (!$new_len) {
break 2;
}
break;
case '+':
++$add;
--$new_len;
$real[] = $line;
break;
case '-':
if (!$old_len) {
// In this case, we've hit "---" from a new file. So don't
// advance the line cursor.
$hit_next_hunk = true;
break 2;
}
++$del;
--$old_len;
$real[] = $line;
break;
case ' ':
if (!$old_len && !$new_len) {
break 2;
}
--$old_len;
--$new_len;
$real[] = $line;
break;
default:
// We hit something, likely another hunk.
$hit_next_hunk = true;
break 2;
}
}
if ($old_len || $new_len) {
$this->didFailParse(pht('Found the wrong number of hunk lines.'));
}
$corpus = implode('', $real);
$is_binary = false;
if ($this->detectBinaryFiles) {
$is_binary = !phutil_is_utf8($corpus);
$try_encoding = $this->tryEncoding;
if ($is_binary && $try_encoding) {
$is_binary = ArcanistDiffUtils::isHeuristicBinaryFile($corpus);
if (!$is_binary) {
$corpus = phutil_utf8_convert($corpus, 'UTF-8', $try_encoding);
if (!phutil_is_utf8($corpus)) {
throw new Exception(pht("Failed to convert a hunk from '%s' to UTF-8. " . "Check that the specified encoding is correct.", $try_encoding));
}
}
}
}
if ($is_binary) {
// SVN happily treats binary files which aren't marked with the right
// mime type as text files. Detect that junk here and mark the file
// binary. We'll catch stuff with unicode too, but that's verboten
// anyway. If there are too many false positives with this we might
// need to make it threshold-triggered instead of triggering on any
// unprintable byte.
$change->setFileType(ArcanistDiffChangeType::FILE_BINARY);
} else {
$hunk->setCorpus($corpus);
$hunk->setAddLines($add);
$hunk->setDelLines($del);
$change->addHunk($hunk);
}
if (!$hit_next_hunk) {
$line = $this->nextNonemptyLine();
}
} while (preg_match('/^@@ /', $line));
}
private function convertNonUTF8Diff($diff)
{
if ($this->encoding) {
$diff = phutil_utf8_convert($diff, $this->encoding, 'UTF-8');
}
return $diff;
}
private function buildPatch(PhabricatorMetaMTAMail $template, PhabricatorRepository $repository, PhabricatorRepositoryCommit $commit)
{
$attach_key = 'metamta.diffusion.attach-patches';
$inline_key = 'metamta.diffusion.inline-patches';
$attach_patches = PhabricatorEnv::getEnvConfig($attach_key);
$inline_patches = PhabricatorEnv::getEnvConfig($inline_key);
if (!$attach_patches && !$inline_patches) {
return;
}
$encoding = $repository->getDetail('encoding', 'UTF-8');
$result = null;
$patch_error = null;
try {
$raw_patch = $this->loadRawPatchText($repository, $commit);
if ($attach_patches) {
$commit_name = $repository->formatCommitName($commit->getCommitIdentifier());
$template->addAttachment(new PhabricatorMetaMTAAttachment($raw_patch, $commit_name . '.patch', 'text/x-patch; charset=' . $encoding));
}
} catch (Exception $ex) {
phlog($ex);
$patch_error = 'Unable to generate: ' . $ex->getMessage();
}
if ($patch_error) {
$result = $patch_error;
} else {
if ($inline_patches) {
$len = substr_count($raw_patch, "\n");
if ($len <= $inline_patches) {
// We send email as utf8, so we need to convert the text to utf8 if
// we can.
if ($encoding) {
$raw_patch = phutil_utf8_convert($raw_patch, 'UTF-8', $encoding);
}
$result = phutil_utf8ize($raw_patch);
}
}
}
if ($result) {
$result = "PATCH\n\n{$result}\n";
}
return $result;
}
}
}
}
$root = dirname(dirname(dirname(__FILE__)));
require_once $root . '/scripts/__init_script__.php';
require_once $root . '/externals/mimemailparser/MimeMailParser.class.php';
$args = new PhutilArgumentParser($argv);
$args->parseStandardArguments();
$args->parse(array(array('name' => 'process-duplicates', 'help' => pht("Process this message, even if it's a duplicate of another message. " . "This is mostly useful when debugging issues with mail routing.")), array('name' => 'env', 'wildcard' => true)));
$parser = new MimeMailParser();
$parser->setText(file_get_contents('php://stdin'));
$text_body = $parser->getMessageBody('text');
$text_body_headers = $parser->getMessageBodyHeaders('text');
$content_type = idx($text_body_headers, 'content-type');
if (!phutil_is_utf8($text_body) && (preg_match('/charset="(.*?)"/', $content_type, $matches) || preg_match('/charset=(\\S+)/', $content_type, $matches))) {
$text_body = phutil_utf8_convert($text_body, 'UTF-8', $matches[1]);
}
$headers = $parser->getHeaders();
$headers['subject'] = iconv_mime_decode($headers['subject'], 0, 'UTF-8');
$headers['from'] = iconv_mime_decode($headers['from'], 0, 'UTF-8');
if ($args->getArg('process-duplicates')) {
$headers['message-id'] = Filesystem::readRandomCharacters(64);
}
$received = new PhabricatorMetaMTAReceivedMail();
$received->setHeaders($headers);
$received->setBodies(array('text' => $text_body, 'html' => $parser->getMessageBody('html')));
$attachments = array();
foreach ($parser->getAttachments() as $attachment) {
if (preg_match('@text/(plain|html)@', $attachment->getContentType()) && $attachment->getContentDisposition() == 'inline') {
// If this is an "inline" attachment with some sort of text content-type,
// do not treat it as a file for attachment. MimeMailParser already picked
private function inlinePatch(PhabricatorMetaMTAMailBody $body, PhabricatorRepositoryCommit $commit)
{
if (!$this->getRawPatch()) {
return;
}
$inline_key = 'metamta.diffusion.inline-patches';
$inline_patches = PhabricatorEnv::getEnvConfig($inline_key);
if (!$inline_patches) {
return;
}
$repository = $commit->getRepository();
$raw_patch = $this->getRawPatch();
$result = null;
$len = substr_count($raw_patch, "\n");
if ($len <= $inline_patches) {
// We send email as utf8, so we need to convert the text to utf8 if
// we can.
$encoding = $repository->getDetail('encoding', 'UTF-8');
if ($encoding) {
$raw_patch = phutil_utf8_convert($raw_patch, 'UTF-8', $encoding);
}
$result = phutil_utf8ize($raw_patch);
}
if ($result) {
$result = "PATCH\n\n{$result}\n";
}
$body->addRawSection($result);
}
protected function validateTransaction(PhabricatorLiskDAO $object, $type, array $xactions)
{
$errors = parent::validateTransaction($object, $type, $xactions);
switch ($type) {
case PhabricatorRepositoryTransaction::TYPE_AUTOCLOSE_ONLY:
case PhabricatorRepositoryTransaction::TYPE_TRACK_ONLY:
foreach ($xactions as $xaction) {
foreach ($xaction->getNewValue() as $pattern) {
// Check for invalid regular expressions.
$regexp = PhabricatorRepository::extractBranchRegexp($pattern);
if ($regexp !== null) {
$ok = @preg_match($regexp, '');
if ($ok === false) {
$error = new PhabricatorApplicationTransactionValidationError($type, pht('Invalid'), pht('Expression "%s" is not a valid regular expression. Note ' . 'that you must include delimiters.', $regexp), $xaction);
$errors[] = $error;
continue;
}
}
// Check for formatting mistakes like `regex(...)` instead of
// `regexp(...)`.
$matches = null;
if (preg_match('/^([^(]+)\\(.*\\)\\z/', $pattern, $matches)) {
switch ($matches[1]) {
case 'regexp':
break;
default:
$error = new PhabricatorApplicationTransactionValidationError($type, pht('Invalid'), pht('Matching function "%s(...)" is not recognized. Valid ' . 'functions are: regexp(...).', $matches[1]), $xaction);
$errors[] = $error;
break;
}
}
}
}
break;
case PhabricatorRepositoryTransaction::TYPE_AUTOMATION_BLUEPRINTS:
foreach ($xactions as $xaction) {
$old = nonempty($xaction->getOldValue(), array());
$new = nonempty($xaction->getNewValue(), array());
$add = array_diff($new, $old);
$invalid = PhabricatorObjectQuery::loadInvalidPHIDsForViewer($this->getActor(), $add);
if ($invalid) {
$errors[] = new PhabricatorApplicationTransactionValidationError($type, pht('Invalid'), pht('Some of the selected automation blueprints are invalid ' . 'or restricted: %s.', implode(', ', $invalid)), $xaction);
}
}
break;
case PhabricatorRepositoryTransaction::TYPE_VCS:
$vcs_map = PhabricatorRepositoryType::getAllRepositoryTypes();
$current_vcs = $object->getVersionControlSystem();
if (!$this->getIsNewObject()) {
foreach ($xactions as $xaction) {
if ($xaction->getNewValue() == $current_vcs) {
continue;
}
$errors[] = new PhabricatorApplicationTransactionValidationError($type, pht('Immutable'), pht('You can not change the version control system an existing ' . 'repository uses. It can only be set when a repository is ' . 'first created.'), $xaction);
}
} else {
$value = $object->getVersionControlSystem();
foreach ($xactions as $xaction) {
$value = $xaction->getNewValue();
if (empty($vcs_map[$value])) {
$errors[] = new PhabricatorApplicationTransactionValidationError($type, pht('Invalid'), pht('Specified version control system must be a VCS ' . 'recognized by Phabricator: %s.', implode(', ', array_keys($vcs_map))), $xaction);
}
}
if (!strlen($value)) {
$error = new PhabricatorApplicationTransactionValidationError($type, pht('Required'), pht('When creating a repository, you must specify a valid ' . 'underlying version control system: %s.', implode(', ', array_keys($vcs_map))), nonempty(last($xactions), null));
$error->setIsMissingFieldError(true);
$errors[] = $error;
}
}
break;
case PhabricatorRepositoryTransaction::TYPE_NAME:
$missing = $this->validateIsEmptyTextField($object->getName(), $xactions);
if ($missing) {
$error = new PhabricatorApplicationTransactionValidationError($type, pht('Required'), pht('Repository name is required.'), nonempty(last($xactions), null));
$error->setIsMissingFieldError(true);
$errors[] = $error;
}
break;
case PhabricatorRepositoryTransaction::TYPE_ACTIVATE:
$status_map = PhabricatorRepository::getStatusMap();
foreach ($xactions as $xaction) {
$status = $xaction->getNewValue();
if (empty($status_map[$status])) {
$errors[] = new PhabricatorApplicationTransactionValidationError($type, pht('Invalid'), pht('Repository status "%s" is not valid.', $status), $xaction);
}
}
break;
case PhabricatorRepositoryTransaction::TYPE_ENCODING:
foreach ($xactions as $xaction) {
// Make sure the encoding is valid by converting to UTF-8. This tests
// that the user has mbstring installed, and also that they didn't
// type a garbage encoding name. Note that we're converting from
// UTF-8 to the target encoding, because mbstring is fine with
// converting from a nonsense encoding.
$encoding = $xaction->getNewValue();
if (!strlen($encoding)) {
continue;
}
try {
phutil_utf8_convert('.', $encoding, 'UTF-8');
} catch (Exception $ex) {
$errors[] = new PhabricatorApplicationTransactionValidationError($type, pht('Invalid'), pht('Repository encoding "%s" is not valid: %s', $encoding, $ex->getMessage()), $xaction);
}
}
break;
case PhabricatorRepositoryTransaction::TYPE_SLUG:
foreach ($xactions as $xaction) {
$old = $xaction->getOldValue();
$new = $xaction->getNewValue();
if (!strlen($new)) {
continue;
}
if ($new === $old) {
continue;
}
try {
PhabricatorRepository::assertValidRepositorySlug($new);
} catch (Exception $ex) {
$errors[] = new PhabricatorApplicationTransactionValidationError($type, pht('Invalid'), $ex->getMessage(), $xaction);
continue;
}
$other = id(new PhabricatorRepositoryQuery())->setViewer(PhabricatorUser::getOmnipotentUser())->withSlugs(array($new))->executeOne();
if ($other && $other->getID() !== $object->getID()) {
$errors[] = new PhabricatorApplicationTransactionValidationError($type, pht('Duplicate'), pht('The selected repository short name is already in use by ' . 'another repository. Choose a unique short name.'), $xaction);
continue;
}
}
break;
case PhabricatorRepositoryTransaction::TYPE_CALLSIGN:
foreach ($xactions as $xaction) {
$old = $xaction->getOldValue();
$new = $xaction->getNewValue();
if (!strlen($new)) {
continue;
}
if ($new === $old) {
continue;
}
try {
PhabricatorRepository::assertValidCallsign($new);
} catch (Exception $ex) {
$errors[] = new PhabricatorApplicationTransactionValidationError($type, pht('Invalid'), $ex->getMessage(), $xaction);
continue;
}
$other = id(new PhabricatorRepositoryQuery())->setViewer(PhabricatorUser::getOmnipotentUser())->withCallsigns(array($new))->executeOne();
if ($other && $other->getID() !== $object->getID()) {
$errors[] = new PhabricatorApplicationTransactionValidationError($type, pht('Duplicate'), pht('The selected callsign ("%s") is already in use by another ' . 'repository. Choose a unique callsign.', $new), $xaction);
continue;
}
}
break;
case PhabricatorRepositoryTransaction::TYPE_SYMBOLS_SOURCES:
foreach ($xactions as $xaction) {
$old = $object->getSymbolSources();
$new = $xaction->getNewValue();
// If the viewer is adding new repositories, make sure they are
// valid and visible.
$add = array_diff($new, $old);
if (!$add) {
continue;
}
$repositories = id(new PhabricatorRepositoryQuery())->setViewer($this->getActor())->withPHIDs($add)->execute();
$repositories = mpull($repositories, null, 'getPHID');
foreach ($add as $phid) {
if (isset($repositories[$phid])) {
continue;
}
$errors[] = new PhabricatorApplicationTransactionValidationError($type, pht('Invalid'), pht('Repository ("%s") does not exist, or you do not have ' . 'permission to see it.', $phid), $xaction);
break;
}
}
break;
}
return $errors;
}
