/**
* Convert a string into valid UTF-8. This function is quite slow.
*
* When invalid byte subsequences are encountered, they will be replaced with
* U+FFFD, the Unicode replacement character.
*
* @param string String to convert to valid UTF-8.
* @return string String with invalid UTF-8 byte subsequences replaced with
* U+FFFD.
*/
function phutil_utf8ize($string)
{
if (phutil_is_utf8($string)) {
return $string;
}
// There is no function to do this in iconv, mbstring or ICU to do this, so
// do it (very very slowly) in pure PHP.
// TODO: Provide an optional fast C implementation ala fb_utf8ize() if this
// ever shows up in profiles?
// NOTE: Overlong 3-byte and 4-byte representations incorrectly survive
// this function.
$result = array();
$regex = "/([-]" . "|[Â-ß][€-¿]" . "|[à-ï][€-¿][€-¿]" . "|[ð-ô][€-¿][€-¿][€-¿])" . "|(.)/";
$offset = 0;
$matches = null;
while (preg_match($regex, $string, $matches, 0, $offset)) {
if (!isset($matches[2])) {
$result[] = $matches[1];
} else {
// Unicode replacement character, U+FFFD.
$result[] = "�";
}
$offset += strlen($matches[0]);
}
return implode('', $result);
}
/**
* Check inserts for characters outside of the BMP. Even with the strictest
* settings, MySQL will silently truncate data when it encounters these, which
* can lead to data loss and security problems.
*/
protected function validateUTF8String($string)
{
if (phutil_is_utf8($string)) {
return;
}
throw new AphrontCharacterSetQueryException(pht('Attempting to construct a query using a non-utf8 string when ' . 'utf8 is expected. Use the `%%B` conversion to escape binary ' . 'strings data.'));
}
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
if ($argc > 1) {
$_SERVER['PHABRICATOR_ENV'] = $argv[1];
}
$root = dirname(dirname(dirname(__FILE__)));
require_once $root . '/scripts/__init_script__.php';
require_once $root . '/externals/mimemailparser/MimeMailParser.class.php';
$parser = new MimeMailParser();
$parser->setText(file_get_contents('php://stdin'));
$text_body = $parser->getMessageBody('text');
$text_body_headers = $parser->getMessageBodyHeaders('text');
$content_type = idx($text_body_headers, 'content-type');
if (!phutil_is_utf8($text_body) && (preg_match('/charset="(.*?)"/', $content_type, $matches) || preg_match('/charset=(\\S+)/', $content_type, $matches))) {
$text_body = phutil_utf8_convert($text_body, "UTF-8", $matches[1]);
}
$headers = $parser->getHeaders();
$headers['subject'] = iconv_mime_decode($headers['subject'], 0, "UTF-8");
$headers['from'] = iconv_mime_decode($headers['from'], 0, "UTF-8");
$received = new PhabricatorMetaMTAReceivedMail();
$received->setHeaders($headers);
$received->setBodies(array('text' => $text_body, 'html' => $parser->getMessageBody('html')));
$attachments = array();
foreach ($parser->getAttachments() as $attachment) {
if (preg_match('@text/(plain|html)@', $attachment->getContentType()) && $attachment->getContentDisposition() == 'inline') {
// If this is an "inline" attachment with some sort of text content-type,
// do not treat it as a file for attachment. MimeMailParser already picked
// it up in the getMessageBody() call above. We still want to treat 'inline'
// attachments with other content types (e.g., images) as attachments.
protected function generateChanges()
{
$parser = $this->newDiffParser();
$is_raw = $this->isRawDiffSource();
if ($is_raw) {
if ($this->getArgument('raw')) {
fwrite(STDERR, pht('Reading diff from stdin...') . "\n");
$raw_diff = file_get_contents('php://stdin');
} else {
if ($this->getArgument('raw-command')) {
list($raw_diff) = execx('%C', $this->getArgument('raw-command'));
} else {
throw new Exception(pht('Unknown raw diff source.'));
}
}
$changes = $parser->parseDiff($raw_diff);
foreach ($changes as $key => $change) {
// Remove "message" changes, e.g. from "git show".
if ($change->getType() == ArcanistDiffChangeType::TYPE_MESSAGE) {
unset($changes[$key]);
}
}
return $changes;
}
$repository_api = $this->getRepositoryAPI();
if ($repository_api instanceof ArcanistSubversionAPI) {
$paths = $this->generateAffectedPaths();
$this->primeSubversionWorkingCopyData($paths);
// Check to make sure the user is diffing from a consistent base revision.
// This is mostly just an abuse sanity check because it's silly to do this
// and makes the code more difficult to effectively review, but it also
// affects patches and makes them nonportable.
$bases = $repository_api->getSVNBaseRevisions();
// Remove all files with baserev "0"; these files are new.
foreach ($bases as $path => $baserev) {
if ($bases[$path] <= 0) {
unset($bases[$path]);
}
}
if ($bases) {
$rev = reset($bases);
$revlist = array();
foreach ($bases as $path => $baserev) {
$revlist[] = ' ' . pht('Revision %s, %s', $baserev, $path);
}
$revlist = implode("\n", $revlist);
foreach ($bases as $path => $baserev) {
if ($baserev !== $rev) {
throw new ArcanistUsageException(pht("Base revisions of changed paths are mismatched. Update all " . "paths to the same base revision before creating a diff: " . "\n\n%s", $revlist));
}
}
// If you have a change which affects several files, all of which are
// at a consistent base revision, treat that revision as the effective
// base revision. The use case here is that you made a change to some
// file, which updates it to HEAD, but want to be able to change it
// again without updating the entire working copy. This is a little
// sketchy but it arises in Facebook Ops workflows with config files and
// doesn't have any real material tradeoffs (e.g., these patches are
// perfectly applyable).
$repository_api->overrideSVNBaseRevisionNumber($rev);
}
$changes = $parser->parseSubversionDiff($repository_api, $paths);
} else {
if ($repository_api instanceof ArcanistGitAPI) {
$diff = $repository_api->getFullGitDiff($repository_api->getBaseCommit(), $repository_api->getHeadCommit());
if (!strlen($diff)) {
throw new ArcanistUsageException(pht('No changes found. (Did you specify the wrong commit range?)'));
}
$changes = $parser->parseDiff($diff);
} else {
if ($repository_api instanceof ArcanistMercurialAPI) {
$diff = $repository_api->getFullMercurialDiff();
if (!strlen($diff)) {
throw new ArcanistUsageException(pht('No changes found. (Did you specify the wrong commit range?)'));
}
$changes = $parser->parseDiff($diff);
} else {
throw new Exception(pht('Repository API is not supported.'));
}
}
}
if (count($changes) > 250) {
$message = pht('This diff has a very large number of changes (%s). Differential ' . 'works best for changes which will receive detailed human review, ' . 'and not as well for large automated changes or bulk checkins. ' . 'See %s for information about reviewing big checkins. Continue anyway?', phutil_count($changes), 'https://secure.phabricator.com/book/phabricator/article/' . 'differential_large_changes/');
if (!phutil_console_confirm($message)) {
throw new ArcanistUsageException(pht('Aborted generation of gigantic diff.'));
}
}
$limit = 1024 * 1024 * 4;
foreach ($changes as $change) {
$size = 0;
foreach ($change->getHunks() as $hunk) {
$size += strlen($hunk->getCorpus());
}
if ($size > $limit) {
$byte_warning = pht("Diff for '%s' with context is %s bytes in length. " . "Generally, source changes should not be this large.", $change->getCurrentPath(), new PhutilNumber($size));
if (!$this->getArgument('less-context')) {
$byte_warning .= ' ' . pht("If this file is a huge text file, try using the '%s' flag.", '--less-context');
}
if ($repository_api instanceof ArcanistSubversionAPI) {
throw new ArcanistUsageException($byte_warning . ' ' . pht("If the file is not a text file, mark it as binary with:" . "\n\n \$ %s\n", 'svn propset svn:mime-type application/octet-stream <filename>'));
} else {
$confirm = $byte_warning . ' ' . pht("If the file is not a text file, you can mark it 'binary'. " . "Mark this file as 'binary' and continue?");
if (phutil_console_confirm($confirm)) {
$change->convertToBinaryChange($repository_api);
} else {
throw new ArcanistUsageException(pht('Aborted generation of gigantic diff.'));
}
}
}
}
$try_encoding = nonempty($this->getArgument('encoding'), null);
$utf8_problems = array();
foreach ($changes as $change) {
foreach ($change->getHunks() as $hunk) {
$corpus = $hunk->getCorpus();
if (!phutil_is_utf8($corpus)) {
// If this corpus is heuristically binary, don't try to convert it.
// mb_check_encoding() and mb_convert_encoding() are both very very
// liberal about what they're willing to process.
$is_binary = ArcanistDiffUtils::isHeuristicBinaryFile($corpus);
if (!$is_binary) {
if (!$try_encoding) {
try {
$try_encoding = $this->getRepositoryEncoding();
} catch (ConduitClientException $e) {
if ($e->getErrorCode() == 'ERR-BAD-ARCANIST-PROJECT') {
echo phutil_console_wrap(pht('Lookup of encoding in arcanist project failed: %s', $e->getMessage()) . "\n");
} else {
throw $e;
}
}
}
if ($try_encoding) {
$corpus = phutil_utf8_convert($corpus, 'UTF-8', $try_encoding);
$name = $change->getCurrentPath();
if (phutil_is_utf8($corpus)) {
$this->writeStatusMessage(pht("Converted a '%s' hunk from '%s' to UTF-8.\n", $name, $try_encoding));
$hunk->setCorpus($corpus);
continue;
}
}
}
$utf8_problems[] = $change;
break;
}
}
}
// If there are non-binary files which aren't valid UTF-8, warn the user
// and treat them as binary changes. See D327 for discussion of why Arcanist
// has this behavior.
if ($utf8_problems) {
$utf8_warning = sprintf("%s\n\n%s\n\n %s\n", pht('This diff includes %s file(s) which are not valid UTF-8 (they ' . 'contain invalid byte sequences). You can either stop this ' . 'workflow and fix these files, or continue. If you continue, ' . 'these files will be marked as binary.', phutil_count($utf8_problems)), pht("You can learn more about how Phabricator handles character " . "encodings (and how to configure encoding settings and detect and " . "correct encoding problems) by reading 'User Guide: UTF-8 and " . "Character Encoding' in the Phabricator documentation."), pht('%s AFFECTED FILE(S)', phutil_count($utf8_problems)));
$confirm = pht('Do you want to mark these %s file(s) as binary and continue?', phutil_count($utf8_problems));
echo phutil_console_format("**%s**\n", pht('Invalid Content Encoding (Non-UTF8)'));
echo phutil_console_wrap($utf8_warning);
$file_list = mpull($utf8_problems, 'getCurrentPath');
$file_list = ' ' . implode("\n ", $file_list);
echo $file_list;
if (!phutil_console_confirm($confirm, $default_no = false)) {
throw new ArcanistUsageException(pht('Aborted workflow to fix UTF-8.'));
} else {
foreach ($utf8_problems as $change) {
$change->convertToBinaryChange($repository_api);
}
}
}
$this->uploadFilesForChanges($changes);
return $changes;
}
/**
* Produce a human-readable explanation why a value can not be JSON-encoded.
*
* @param wild Value to validate.
* @param string Path within the object to provide context.
* @return string|null Explanation of why it can't be encoded, or null.
*/
function phutil_validate_json($value, $path = '')
{
if ($value === null) {
return;
}
if ($value === true) {
return;
}
if ($value === false) {
return;
}
if (is_int($value)) {
return;
}
if (is_float($value)) {
return;
}
if (is_array($value)) {
foreach ($value as $key => $subvalue) {
if (strlen($path)) {
$full_key = $path . ' > ';
} else {
$full_key = '';
}
if (!phutil_is_utf8($key)) {
$full_key = $full_key . phutil_utf8ize($key);
return pht('Dictionary key "%s" is not valid UTF8, and can not be JSON encoded.', $full_key);
}
$full_key .= $key;
$result = phutil_validate_json($subvalue, $full_key);
if ($result !== null) {
return $result;
}
}
}
if (is_string($value)) {
if (!phutil_is_utf8($value)) {
$display = substr($value, 0, 256);
$display = phutil_utf8ize($display);
if (!strlen($path)) {
return pht('String value is not valid UTF8, and can not be JSON encoded: %s', $display);
} else {
return pht('Dictionary value at key "%s" is not valid UTF8, and can not be ' . 'JSON encoded: %s', $path, $display);
}
}
}
return;
}
protected function generateChanges()
{
$parser = $this->newDiffParser();
$is_raw = $this->isRawDiffSource();
if ($is_raw) {
if ($this->getArgument('raw')) {
file_put_contents('php://stderr', "Reading diff from stdin...\n");
$raw_diff = file_get_contents('php://stdin');
} else {
if ($this->getArgument('raw-command')) {
list($raw_diff) = execx($this->getArgument('raw-command'));
} else {
throw new Exception("Unknown raw diff source.");
}
}
$changes = $parser->parseDiff($raw_diff);
foreach ($changes as $key => $change) {
// Remove "message" changes, e.g. from "git show".
if ($change->getType() == ArcanistDiffChangeType::TYPE_MESSAGE) {
unset($changes[$key]);
}
}
return $changes;
}
$repository_api = $this->getRepositoryAPI();
if ($repository_api instanceof ArcanistSubversionAPI) {
$paths = $this->generateAffectedPaths();
$this->primeSubversionWorkingCopyData($paths);
// Check to make sure the user is diffing from a consistent base revision.
// This is mostly just an abuse sanity check because it's silly to do this
// and makes the code more difficult to effectively review, but it also
// affects patches and makes them nonportable.
$bases = $repository_api->getSVNBaseRevisions();
// Remove all files with baserev "0"; these files are new.
foreach ($bases as $path => $baserev) {
if ($bases[$path] <= 0) {
unset($bases[$path]);
}
}
if ($bases) {
$rev = reset($bases);
$revlist = array();
foreach ($bases as $path => $baserev) {
$revlist[] = " Revision {$baserev}, {$path}";
}
$revlist = implode("\n", $revlist);
foreach ($bases as $path => $baserev) {
if ($baserev !== $rev) {
throw new ArcanistUsageException("Base revisions of changed paths are mismatched. Update all " . "paths to the same base revision before creating a diff: " . "\n\n" . $revlist);
}
}
// If you have a change which affects several files, all of which are
// at a consistent base revision, treat that revision as the effective
// base revision. The use case here is that you made a change to some
// file, which updates it to HEAD, but want to be able to change it
// again without updating the entire working copy. This is a little
// sketchy but it arises in Facebook Ops workflows with config files and
// doesn't have any real material tradeoffs (e.g., these patches are
// perfectly applyable).
$repository_api->overrideSVNBaseRevisionNumber($rev);
}
$changes = $parser->parseSubversionDiff($repository_api, $paths);
} else {
if ($repository_api instanceof ArcanistGitAPI) {
$diff = $repository_api->getFullGitDiff();
if (!strlen($diff)) {
throw new ArcanistUsageException("No changes found. (Did you specify the wrong commit range?)");
}
$changes = $parser->parseDiff($diff);
} else {
if ($repository_api instanceof ArcanistMercurialAPI) {
$diff = $repository_api->getFullMercurialDiff();
if (!strlen($diff)) {
throw new ArcanistUsageException("No changes found. (Did you specify the wrong commit range?)");
}
$changes = $parser->parseDiff($diff);
} else {
throw new Exception("Repository API is not supported.");
}
}
}
if (count($changes) > 250) {
$count = number_format(count($changes));
$message = "This diff has a very large number of changes ({$count}). " . "Differential works best for changes which will receive detailed " . "human review, and not as well for large automated changes or " . "bulk checkins. Continue anyway?";
if (!phutil_console_confirm($message)) {
throw new ArcanistUsageException("Aborted generation of gigantic diff.");
}
}
$limit = 1024 * 1024 * 4;
foreach ($changes as $change) {
$size = 0;
foreach ($change->getHunks() as $hunk) {
$size += strlen($hunk->getCorpus());
}
if ($size > $limit) {
$file_name = $change->getCurrentPath();
$change_size = number_format($size);
$byte_warning = "Diff for '{$file_name}' with context is {$change_size} bytes in " . "length. Generally, source changes should not be this large.";
if (!$this->getArgument('less-context')) {
$byte_warning .= " If this file is a huge text file, try using the " . "'--less-context' flag.";
}
if ($repository_api instanceof ArcanistSubversionAPI) {
throw new ArcanistUsageException("{$byte_warning} If the file is not a text file, mark it as " . "binary with:" . "\n\n" . " \$ svn propset svn:mime-type application/octet-stream <filename>" . "\n");
} else {
$confirm = "{$byte_warning} If the file is not a text file, you can " . "mark it 'binary'. Mark this file as 'binary' and continue?";
if (phutil_console_confirm($confirm)) {
$change->convertToBinaryChange();
} else {
throw new ArcanistUsageException("Aborted generation of gigantic diff.");
}
}
}
}
$try_encoding = nonempty($this->getArgument('encoding'), null);
$utf8_problems = array();
foreach ($changes as $change) {
foreach ($change->getHunks() as $hunk) {
$corpus = $hunk->getCorpus();
if (!phutil_is_utf8($corpus)) {
// If this corpus is heuristically binary, don't try to convert it.
// mb_check_encoding() and mb_convert_encoding() are both very very
// liberal about what they're willing to process.
$is_binary = ArcanistDiffUtils::isHeuristicBinaryFile($corpus);
if (!$is_binary) {
if (!$try_encoding) {
try {
$try_encoding = $this->getRepositoryEncoding();
} catch (ConduitClientException $e) {
if ($e->getErrorCode() == 'ERR-BAD-ARCANIST-PROJECT') {
echo phutil_console_wrap("Lookup of encoding in arcanist project failed\n" . $e->getMessage());
} else {
throw $e;
}
}
}
if ($try_encoding && $try_encoding != 'UTF-8') {
if (!function_exists('mb_convert_encoding')) {
throw new ArcanistUsageException("This diff includes a file encoded in '{$try_encoding}', " . "but you don't have the PHP mbstring extension installed " . "so it can't be converted to UTF-8. Install mbstring.");
}
$corpus = mb_convert_encoding($corpus, 'UTF-8', $try_encoding);
$name = $change->getCurrentPath();
if (phutil_is_utf8($corpus)) {
$this->writeStatusMessage("Converted a '{$name}' hunk from '{$try_encoding}' " . "to UTF-8.\n");
$hunk->setCorpus($corpus);
continue;
}
}
}
$utf8_problems[] = $change;
break;
}
}
}
// If there are non-binary files which aren't valid UTF-8, warn the user
// and treat them as binary changes. See D327 for discussion of why Arcanist
// has this behavior.
if ($utf8_problems) {
$utf8_warning = pht("This diff includes file(s) which are not valid UTF-8 (they contain " . "invalid byte sequences). You can either stop this workflow and " . "fix these files, or continue. If you continue, these files will " . "be marked as binary.", count($utf8_problems)) . "\n\n" . "You can learn more about how Phabricator handles character encodings " . "(and how to configure encoding settings and detect and correct " . "encoding problems) by reading 'User Guide: UTF-8 and Character " . "Encoding' in the Phabricator documentation.\n\n";
" " . pht('AFFECTED FILE(S)', count($utf8_problems)) . "\n";
$confirm = pht('Do you want to mark these files as binary and continue?', count($utf8_problems));
echo phutil_console_format("**Invalid Content Encoding (Non-UTF8)**\n");
echo phutil_console_wrap($utf8_warning);
$file_list = mpull($utf8_problems, 'getCurrentPath');
$file_list = ' ' . implode("\n ", $file_list);
echo $file_list;
if (!phutil_console_confirm($confirm, $default_no = false)) {
throw new ArcanistUsageException("Aborted workflow to fix UTF-8.");
} else {
foreach ($utf8_problems as $change) {
$change->convertToBinaryChange();
}
}
}
foreach ($changes as $change) {
$path = $change->getCurrentPath();
// Certain types of changes (moves and copies) don't contain change data
// when expressed in raw "git diff" form. Augment any such diffs with
// textual data.
if ($change->getNeedsSyntheticGitHunks()) {
$diff = $repository_api->getRawDiffText($path, $moves = false);
$parser = $this->newDiffParser();
$raw_changes = $parser->parseDiff($diff);
foreach ($raw_changes as $raw_change) {
if ($raw_change->getCurrentPath() == $path) {
$change->setFileType($raw_change->getFileType());
foreach ($raw_change->getHunks() as $hunk) {
$change->addHunk($hunk);
}
break;
}
}
$change->setNeedsSyntheticGitHunks(false);
}
if ($change->getFileType() != ArcanistDiffChangeType::FILE_BINARY) {
continue;
}
$name = basename($path);
$old_file = $repository_api->getOriginalFileData($path);
$old_dict = $this->uploadFile($old_file, $name, 'old binary');
if ($old_dict['guid']) {
$change->setMetadata('old:binary-phid', $old_dict['guid']);
}
$change->setMetadata('old:file:size', $old_dict['size']);
$change->setMetadata('old:file:mime-type', $old_dict['mime']);
$new_file = $repository_api->getCurrentFileData($path);
$new_dict = $this->uploadFile($new_file, $name, 'new binary');
if ($new_dict['guid']) {
$change->setMetadata('new:binary-phid', $new_dict['guid']);
}
$change->setMetadata('new:file:size', $new_dict['size']);
$change->setMetadata('new:file:mime-type', $new_dict['mime']);
if (preg_match('@^image/@', $new_dict['mime'])) {
$change->setFileType(ArcanistDiffChangeType::FILE_IMAGE);
}
}
return $changes;
}
protected function detectEncodingForStorage($string)
{
return phutil_is_utf8($string) ? 'utf8' : null;
}
protected function parseChangeset(ArcanistDiffChange $change)
{
// If a diff includes two sets of changes to the same file, let the
// second one win. In particular, this occurs when adding subdirectories
// in Subversion that contain files: the file text will be present in
// both the directory diff and the file diff. See T5555. Dropping the
// hunks lets whichever one shows up later win instead of showing changes
// twice.
$change->dropHunks();
$all_changes = array();
do {
$hunk = new ArcanistDiffHunk();
$line = $this->getLineTrimmed();
$real = array();
// In the case where only one line is changed, the length is omitted.
// The final group is for git, which appends a guess at the function
// context to the diff.
$matches = null;
$ok = preg_match('/^@@ -(\\d+)(?:,(\\d+))? \\+(\\d+)(?:,(\\d+))? @@(?: .*?)?$/U', $line, $matches);
if (!$ok) {
// It's possible we hit the style of an svn1.7 property change.
// This is a 4-line Index block, followed by an empty line, followed
// by a "Property changes on:" section similar to svn1.6.
if ($line == '') {
$line = $this->nextNonemptyLine();
$ok = preg_match('/^Property changes on:/', $line);
if (!$ok) {
$this->didFailParse(pht('Confused by empty line'));
}
$line = $this->nextLine();
return $this->parsePropertyHunk($change);
}
$this->didFailParse(pht("Expected hunk header '%s'.", '@@ -NN,NN +NN,NN @@'));
}
$hunk->setOldOffset($matches[1]);
$hunk->setNewOffset($matches[3]);
// Cover for the cases where length wasn't present (implying one line).
$old_len = idx($matches, 2);
if (!strlen($old_len)) {
$old_len = 1;
}
$new_len = idx($matches, 4);
if (!strlen($new_len)) {
$new_len = 1;
}
$hunk->setOldLength($old_len);
$hunk->setNewLength($new_len);
$add = 0;
$del = 0;
$hit_next_hunk = false;
while (($line = $this->nextLine()) !== null) {
if (strlen(rtrim($line, "\r\n"))) {
$char = $line[0];
} else {
// Normally, we do not encouter empty lines in diffs, because
// unchanged lines have an initial space. However, in Git, with
// the option `diff.suppress-blank-empty` set, unchanged blank lines
// emit as completely empty. If we encounter a completely empty line,
// treat it as a ' ' (i.e., unchanged empty line) line.
$char = ' ';
}
switch ($char) {
case '\\':
if (!preg_match('@\\ No newline at end of file@', $line)) {
$this->didFailParse(pht("Expected '\\ No newline at end of file'."));
}
if ($new_len) {
$real[] = $line;
$hunk->setIsMissingOldNewline(true);
} else {
$real[] = $line;
$hunk->setIsMissingNewNewline(true);
}
if (!$new_len) {
break 2;
}
break;
case '+':
++$add;
--$new_len;
$real[] = $line;
break;
case '-':
if (!$old_len) {
// In this case, we've hit "---" from a new file. So don't
// advance the line cursor.
$hit_next_hunk = true;
break 2;
}
++$del;
--$old_len;
$real[] = $line;
break;
case ' ':
if (!$old_len && !$new_len) {
break 2;
}
--$old_len;
--$new_len;
$real[] = $line;
break;
default:
// We hit something, likely another hunk.
$hit_next_hunk = true;
break 2;
}
}
if ($old_len || $new_len) {
$this->didFailParse(pht('Found the wrong number of hunk lines.'));
}
$corpus = implode('', $real);
$is_binary = false;
if ($this->detectBinaryFiles) {
$is_binary = !phutil_is_utf8($corpus);
$try_encoding = $this->tryEncoding;
if ($is_binary && $try_encoding) {
$is_binary = ArcanistDiffUtils::isHeuristicBinaryFile($corpus);
if (!$is_binary) {
$corpus = phutil_utf8_convert($corpus, 'UTF-8', $try_encoding);
if (!phutil_is_utf8($corpus)) {
throw new Exception(pht("Failed to convert a hunk from '%s' to UTF-8. " . "Check that the specified encoding is correct.", $try_encoding));
}
}
}
}
if ($is_binary) {
// SVN happily treats binary files which aren't marked with the right
// mime type as text files. Detect that junk here and mark the file
// binary. We'll catch stuff with unicode too, but that's verboten
// anyway. If there are too many false positives with this we might
// need to make it threshold-triggered instead of triggering on any
// unprintable byte.
$change->setFileType(ArcanistDiffChangeType::FILE_BINARY);
} else {
$hunk->setCorpus($corpus);
$hunk->setAddLines($add);
$hunk->setDelLines($del);
$change->addHunk($hunk);
}
if (!$hit_next_hunk) {
$line = $this->nextNonemptyLine();
}
} while (preg_match('/^@@ /', $line));
}
public function testUTF8BMP()
{
$tests = array("" => array(true, true, "empty string"), "a" => array(true, true, "a"), "a͠͠" => array(true, true, "a with combining"), "☃" => array(true, true, "snowman"), "" => array(true, true, "U+FFFF"), "��" => array(false, false, "Invalid, byte range."), "𐀀" => array(true, false, "U+10000"), "𝄞" => array(true, false, "gclef"), "musical 𝄞 g-clef" => array(true, false, "gclef text"), "�" => array(false, false, "Invalid, truncated."), "���" => array(false, false, "Nonminimal 3-byte character."), "�" => array(false, false, "Partial 2-byte character."), "�" => array(false, false, "Partial BMP 0xE0 character."), "�" => array(false, false, "Partial BMP cahracter."));
foreach ($tests as $input => $test) {
list($expect_utf8, $expect_bmp, $test_name) = $test;
$this->assertEqual($expect_utf8, phutil_is_utf8($input), pht('is_utf(%s)', $test_name));
$this->assertEqual($expect_bmp, phutil_is_utf8_with_only_bmp_characters($input), pht('is_utf_bmp(%s)', $test_name));
}
}
private function buildCorpus($selected, $file_query, $needs_blame, $drequest, $path, $data)
{
$image_type = $this->getImageType($path);
if ($image_type && !$selected) {
$corpus = phutil_render_tag('img', array('style' => 'padding-bottom: 10px', 'src' => 'data:' . $image_type . ';base64,' . base64_encode($data)));
return $corpus;
}
$document_type = $this->getDocumentType($path);
if ($document_type && !$selected || !phutil_is_utf8($data)) {
$data = $file_query->getRawData();
$document_type_description = $document_type ? $document_type : 'binary';
$corpus = phutil_render_tag('p', array('style' => 'text-align: center;'), phutil_render_tag('a', array('href' => '?view=raw', 'class' => 'button'), "View {$document_type_description}"));
return $corpus;
}
// TODO: blame of blame.
switch ($selected) {
case 'plain':
$style = "margin: 1em 2em; width: 90%; height: 80em; font-family: monospace";
$corpus = phutil_render_tag('textarea', array('style' => $style), phutil_escape_html($file_query->getRawData()));
break;
case 'plainblame':
$style = "margin: 1em 2em; width: 90%; height: 80em; font-family: monospace";
list($text_list, $rev_list, $blame_dict) = $file_query->getBlameData();
$rows = array();
foreach ($text_list as $k => $line) {
$rev = $rev_list[$k];
if (isset($blame_dict[$rev]['handle'])) {
$author = $blame_dict[$rev]['handle']->getName();
} else {
$author = $blame_dict[$rev]['author'];
}
$rows[] = sprintf("%-10s %-20s %s", substr($rev, 0, 7), $author, $line);
}
$corpus = phutil_render_tag('textarea', array('style' => $style), phutil_escape_html(implode("\n", $rows)));
break;
case 'highlighted':
case 'blame':
default:
require_celerity_resource('syntax-highlighting-css');
list($text_list, $rev_list, $blame_dict) = $file_query->getBlameData();
$text_list = implode("\n", $text_list);
$text_list = PhabricatorSyntaxHighlighter::highlightWithFilename($path, $text_list);
$text_list = explode("\n", $text_list);
$rows = $this->buildDisplayRows($text_list, $rev_list, $blame_dict, $needs_blame, $drequest, $file_query, $selected);
$corpus_table = phutil_render_tag('table', array('class' => "diffusion-source remarkup-code PhabricatorMonospaced"), implode("\n", $rows));
$corpus = phutil_render_tag('div', array('style' => 'padding: 0pt 2em;'), $corpus_table);
break;
}
return $corpus;
}
function show(array $files, $context)
{
foreach ($files as $file) {
$data = read($file);
$ok = phutil_is_utf8($data);
if ($ok) {
echo "OKAY";
} else {
echo "FAIL";
}
echo " " . name($file) . "\n";
if (!$ok) {
$lines = explode("\n", $data);
$len = count($lines);
$map = array();
$bad = array();
foreach ($lines as $n => $line) {
if (phutil_is_utf8($line)) {
continue;
}
$bad[$n] = true;
for ($jj = max(0, $n - $context); $jj < min($len, $n + 1 + $context); $jj++) {
$map[$jj] = true;
}
}
$width = strlen(max(array_keys($map)));
// Set $last such that we print a newline on the first iteration thorugh
// the loop.
$last = -2;
foreach ($map as $idx => $ignored) {
if ($idx != $last + 1) {
printf("\n");
}
$last = $idx;
$line = $lines[$idx];
if (!empty($bad[$idx])) {
$line = show_problems($line);
}
printf(" % {$width}d %s\n", $idx + 1, $line);
}
echo "\n";
}
}
return 0;
}
protected function parseChangeset(ArcanistDiffChange $change)
{
$all_changes = array();
do {
$hunk = new ArcanistDiffHunk();
$line = $this->getLine();
$real = array();
// In the case where only one line is changed, the length is omitted.
// The final group is for git, which appends a guess at the function
// context to the diff.
$matches = null;
$ok = preg_match('/^@@ -(\\d+)(?:,(\\d+))? \\+(\\d+)(?:,(\\d+))? @@(?: .*?)?$/U', $line, $matches);
if (!$ok) {
// It's possible we hit the style of an svn1.7 property change.
// This is a 4-line Index block, followed by an empty line, followed
// by a "Property changes on:" section similar to svn1.6.
if ($line == '') {
$line = $this->nextNonemptyLine();
$ok = preg_match('/^Property changes on:/', $line);
if (!$ok) {
$this->didFailParse("Confused by empty line");
}
$line = $this->nextLine();
return $this->parsePropertyHunk($change);
}
$this->didFailParse("Expected hunk header '@@ -NN,NN +NN,NN @@'.");
}
$hunk->setOldOffset($matches[1]);
$hunk->setNewOffset($matches[3]);
// Cover for the cases where length wasn't present (implying one line).
$old_len = idx($matches, 2);
if (!strlen($old_len)) {
$old_len = 1;
}
$new_len = idx($matches, 4);
if (!strlen($new_len)) {
$new_len = 1;
}
$hunk->setOldLength($old_len);
$hunk->setNewLength($new_len);
$add = 0;
$del = 0;
$advance = false;
while (($line = $this->nextLine()) !== null) {
if (strlen($line)) {
$char = $line[0];
} else {
$char = '~';
}
switch ($char) {
case '\\':
if (!preg_match('@\\ No newline at end of file@', $line)) {
$this->didFailParse("Expected '\\ No newline at end of file'.");
}
if ($new_len) {
$real[] = $line;
$hunk->setIsMissingOldNewline(true);
} else {
$real[] = $line;
$hunk->setIsMissingNewNewline(true);
}
if (!$new_len) {
$advance = true;
break 2;
}
break;
case '+':
if (!$new_len) {
break 2;
}
++$add;
--$new_len;
$real[] = $line;
break;
case '-':
if (!$old_len) {
break 2;
}
++$del;
--$old_len;
$real[] = $line;
break;
case ' ':
if (!$old_len && !$new_len) {
break 2;
}
--$old_len;
--$new_len;
$real[] = $line;
break;
case '~':
$advance = true;
break 2;
default:
break 2;
}
}
if ($old_len != 0 || $new_len != 0) {
$this->didFailParse("Found the wrong number of hunk lines.");
}
$corpus = implode("\n", $real);
$is_binary = false;
if ($this->detectBinaryFiles) {
$is_binary = !phutil_is_utf8($corpus);
if ($is_binary && $this->tryEncoding) {
$is_binary = ArcanistDiffUtils::isHeuristicBinaryFile($corpus);
if (!$is_binary) {
// NOTE: This feature is HIGHLY EXPERIMENTAL and will cause a lot
// of issues. Use it at your own risk.
$corpus = mb_convert_encoding($corpus, 'UTF-8', $this->tryEncoding);
if (!phutil_is_utf8($corpus)) {
throw new Exception('Failed converting hunk to ' . $this->tryEncoding);
}
}
}
}
if ($is_binary) {
// SVN happily treats binary files which aren't marked with the right
// mime type as text files. Detect that junk here and mark the file
// binary. We'll catch stuff with unicode too, but that's verboten
// anyway. If there are too many false positives with this we might
// need to make it threshold-triggered instead of triggering on any
// unprintable byte.
$change->setFileType(ArcanistDiffChangeType::FILE_BINARY);
} else {
$hunk->setCorpus($corpus);
$hunk->setAddLines($add);
$hunk->setDelLines($del);
$change->addHunk($hunk);
}
if ($advance) {
$line = $this->nextNonemptyLine();
}
} while (preg_match('/^@@ /', $line));
}
public function testUTF8BMP()
{
$tests = array('' => array(true, true, pht('empty string')), 'a' => array(true, true, 'a'), "a͠͠" => array(true, true, pht('%s with combining', 'a')), "☃" => array(true, true, pht('snowman')), "" => array(true, true, 'U+FFFF'), "��" => array(false, false, pht('Invalid, byte range.')), "����" => array(false, false, pht('Nonminimal 4-byte character.')), "𐀀" => array(true, false, 'U+10000'), "𝄞" => array(true, false, 'gclef'), "musical 𝄞 g-clef" => array(true, false, pht('gclef text')), "�" => array(false, false, pht('Invalid, truncated.')), "���" => array(false, false, pht('Nonminimal 3-byte character.')), "�" => array(false, false, pht('Partial 2-byte character.')), "�" => array(false, false, pht('Partial BMP 0xE0 character.')), "�" => array(false, false, pht('Partial BMP cahracter.')));
foreach ($tests as $input => $test) {
list($expect_utf8, $expect_bmp, $test_name) = $test;
// Depending on what's installed on the system, this may use an
// extension.
$this->assertEqual($expect_utf8, phutil_is_utf8($input), pht('is_utf(%s)', $test_name));
// Also test this against the pure PHP implementation, explicitly.
$this->assertEqual($expect_utf8, phutil_is_utf8_slowly($input), pht('is_utf_slowly(%s)', $test_name));
$this->assertEqual($expect_bmp, phutil_is_utf8_with_only_bmp_characters($input), pht('is_utf_bmp(%s)', $test_name));
}
}
