No OneTemporary
Actions

Size

171 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/scripts/symbols/generate_ctags_symbols.php b/scripts/symbols/generate_ctags_symbols.php
	index 6c52521d7e..c70c408bd6 100755
	--- a/scripts/symbols/generate_ctags_symbols.php
	+++ b/scripts/symbols/generate_ctags_symbols.php
	@@ -1,124 +1,126 @@
	#!/usr/bin/env php
	<?php

	$root = dirname(dirname(dirname(__FILE__)));
	require_once $root.'/scripts/__init_script__.php';

	if (ctags_check_executable() == false) {
	echo phutil_console_format(
	"Could not find Exuberant ctags. Make sure it is installed and\n".
	"available in executable path.\n\n".
	"Exuberant ctags project page: http://ctags.sourceforge.net/\n");
	exit(1);
	}

	if ($argc !== 1 \|\| posix_isatty(STDIN)) {
	echo phutil_console_format(
	"usage: find . -type f -name '*.py' \| ./generate_ctags_symbols.php\n");
	exit(1);
	}

	$input = file_get_contents('php://stdin');
	$input = trim($input);
	$input = explode("\n", $input);

	$data = array();
	$futures = array();

	foreach ($input as $file) {
	$file = Filesystem::readablePath($file);
	$futures[$file] = ctags_get_parser_future($file);
	}

	-foreach (Futures($futures)->limit(8) as $file => $future) {
	+$futures = id(new FutureIterator($futures))
	+ ->limit(8);
	+foreach ($futures as $file => $future) {
	$tags = $future->resolve();
	$tags = explode("\n", $tags[1]);

	foreach ($tags as $tag) {
	$parts = explode(';', $tag);
	// skip lines that we can not parse
	if (count($parts) < 2) {
	continue;
	}

	// split ctags information
	$tag_info = explode("\t", $parts[0]);
	// split exuberant ctags "extension fields" (additional information)
	$parts[1] = trim($parts[1], "\t \"");
	$extension_fields = explode("\t", $parts[1]);

	// skip lines that we can not parse
	if (count($tag_info) < 3 \|\| count($extension_fields) < 2) {
	continue;
	}

	// default $context to empty
	$extension_fields[] = '';
	list($token, $file_path, $line_num) = $tag_info;
	list($type, $language, $context) = $extension_fields;

	// skip lines with tokens containing a space
	if (strpos($token, ' ') !== false) {
	continue;
	}

	// strip "language:"
	$language = substr($language, 9);

	// To keep consistent with "Separate with commas, for example: php, py"
	// in Arcanist Project edit form.
	$language = str_ireplace('python', 'py', $language);

	// also, "normalize" c++ and c#
	$language = str_ireplace('c++', 'cpp', $language);
	$language = str_ireplace('c#', 'cs', $language);

	// Ruby has "singleton method", for example
	$type = substr(str_replace(' ', '_', $type), 0, 12);
	// class:foo, struct:foo, union:foo, enum:foo, ...
	$context = last(explode(':', $context, 2));

	$ignore = array(
	'variable' => true,
	);
	if (empty($ignore[$type])) {
	print_symbol($file_path, $line_num, $type, $token, $context, $language);
	}
	}
	}

	function ctags_get_parser_future($file_path) {
	$future = new ExecFuture('ctags -n --fields=Kls -o - %s',
	$file_path);
	return $future;
	}

	function ctags_check_executable() {
	$future = new ExecFuture('ctags --version');
	$result = $future->resolve();

	if (empty($result[1])) {
	return false;
	}

	return true;
	}

	function print_symbol($file, $line_num, $type, $token, $context, $language) {
	// get rid of relative path
	$file = explode('/', $file);
	if ($file[0] == '.' \|\| $file[0] == '..') {
	array_shift($file);
	}
	$file = '/'.implode('/', $file);

	$parts = array(
	$context,
	$token,
	$type,
	strtolower($language),
	$line_num,
	$file,
	);
	echo implode(' ', $parts)."\n";
	}
	diff --git a/scripts/symbols/generate_php_symbols.php b/scripts/symbols/generate_php_symbols.php
	index 087898fdda..4129af557c 100755
	--- a/scripts/symbols/generate_php_symbols.php
	+++ b/scripts/symbols/generate_php_symbols.php
	@@ -1,113 +1,115 @@
	#!/usr/bin/env php
	<?php

	$root = dirname(dirname(dirname(__FILE__)));
	require_once $root.'/scripts/__init_script__.php';

	if ($argc !== 1 \|\| posix_isatty(STDIN)) {
	echo phutil_console_format(
	"usage: find . -type f -name '*.php' \| ./generate_php_symbols.php\n");
	exit(1);
	}

	$input = file_get_contents('php://stdin');
	$input = trim($input);
	$input = explode("\n", $input);

	$data = array();
	$futures = array();

	foreach ($input as $file) {
	$file = Filesystem::readablePath($file);
	$data[$file] = Filesystem::readFile($file);
	$futures[$file] = xhpast_get_parser_future($data[$file]);
	}

	-foreach (Futures($futures)->limit(8) as $file => $future) {
	+$futures = id(new FutureIterator($futures))
	+ ->limit(8);
	+foreach ($futures as $file => $future) {
	$tree = XHPASTTree::newFromDataAndResolvedExecFuture(
	$data[$file],
	$future->resolve());

	$root = $tree->getRootNode();
	$scopes = array();

	$functions = $root->selectDescendantsOfType('n_FUNCTION_DECLARATION');
	foreach ($functions as $function) {
	$name = $function->getChildByIndex(2);
	// Skip anonymous functions
	if (!$name->getConcreteString()) {
	continue;
	}
	print_symbol($file, 'function', $name);
	}

	$classes = $root->selectDescendantsOfType('n_CLASS_DECLARATION');
	foreach ($classes as $class) {
	$class_name = $class->getChildByIndex(1);
	print_symbol($file, 'class', $class_name);
	$scopes[] = array($class, $class_name);
	}

	$interfaces = $root->selectDescendantsOfType('n_INTERFACE_DECLARATION');
	foreach ($interfaces as $interface) {
	$interface_name = $interface->getChildByIndex(1);
	// We don't differentiate classes and interfaces in highlighters.
	print_symbol($file, 'class', $interface_name);
	$scopes[] = array($interface, $interface_name);
	}

	$constants = $root->selectDescendantsOfType('n_CONSTANT_DECLARATION_LIST');
	foreach ($constants as $constant_list) {
	foreach ($constant_list->getChildren() as $constant) {
	$constant_name = $constant->getChildByIndex(0);
	print_symbol($file, 'constant', $constant_name);
	}
	}

	foreach ($scopes as $scope) {
	// this prints duplicate symbols in the case of nested classes
	// luckily, PHP doesn't allow those
	list($class, $class_name) = $scope;

	$consts = $class->selectDescendantsOfType(
	'n_CLASS_CONSTANT_DECLARATION_LIST');
	foreach ($consts as $const_list) {
	foreach ($const_list->getChildren() as $const) {
	$const_name = $const->getChildByIndex(0);
	print_symbol($file, 'class_const', $const_name, $class_name);
	}
	}

	$members = $class->selectDescendantsOfType(
	'n_CLASS_MEMBER_DECLARATION_LIST');
	foreach ($members as $member_list) {
	foreach ($member_list->getChildren() as $member) {
	if ($member->getTypeName() == 'n_CLASS_MEMBER_MODIFIER_LIST') {
	continue;
	}
	$member_name = $member->getChildByIndex(0);
	print_symbol($file, 'member', $member_name, $class_name);
	}
	}

	$methods = $class->selectDescendantsOfType('n_METHOD_DECLARATION');
	foreach ($methods as $method) {
	$method_name = $method->getChildByIndex(2);
	print_symbol($file, 'method', $method_name, $class_name);
	}
	}
	}

	function print_symbol($file, $type, $token, $context = null) {
	$parts = array(
	$context ? $context->getConcreteString() : '',
	// variable tokens are `$name`, not just `name`, so strip the $ off of
	// class field names
	ltrim($token->getConcreteString(), '$'),
	$type,
	'php',
	$token->getLineNumber(),
	'/'.ltrim($file, './'),
	);
	echo implode(' ', $parts)."\n";
	}
	diff --git a/src/applications/differential/parser/DifferentialChangesetParser.php b/src/applications/differential/parser/DifferentialChangesetParser.php
	index 9db301ab35..a27f8c823b 100644
	--- a/src/applications/differential/parser/DifferentialChangesetParser.php
	+++ b/src/applications/differential/parser/DifferentialChangesetParser.php
	@@ -1,1297 +1,1297 @@
	<?php

	final class DifferentialChangesetParser {

	protected $visible = array();
	protected $new = array();
	protected $old = array();
	protected $intra = array();
	protected $newRender = null;
	protected $oldRender = null;

	protected $filename = null;
	protected $hunkStartLines = array();

	protected $comments = array();
	protected $specialAttributes = array();

	protected $changeset;
	protected $whitespaceMode = null;

	protected $renderCacheKey = null;

	private $handles = array();
	private $user;

	private $leftSideChangesetID;
	private $leftSideAttachesToNewFile;

	private $rightSideChangesetID;
	private $rightSideAttachesToNewFile;

	private $originalLeft;
	private $originalRight;

	private $renderingReference;
	private $isSubparser;

	private $isTopLevel;
	private $coverage;
	private $markupEngine;
	private $highlightErrors;
	private $disableCache;
	private $renderer;
	private $characterEncoding;
	private $highlightAs;

	public function setHighlightAs($highlight_as) {
	$this->highlightAs = $highlight_as;
	return $this;
	}

	public function getHighlightAs() {
	return $this->highlightAs;
	}

	public function setCharacterEncoding($character_encoding) {
	$this->characterEncoding = $character_encoding;
	return $this;
	}

	public function getCharacterEncoding() {
	return $this->characterEncoding;
	}

	public function setRenderer($renderer) {
	$this->renderer = $renderer;
	return $this;
	}

	public function getRenderer() {
	if (!$this->renderer) {
	return new DifferentialChangesetTwoUpRenderer();
	}
	return $this->renderer;
	}

	public function setDisableCache($disable_cache) {
	$this->disableCache = $disable_cache;
	return $this;
	}

	public function getDisableCache() {
	return $this->disableCache;
	}

	const CACHE_VERSION = 11;
	const CACHE_MAX_SIZE = 8e6;

	const ATTR_GENERATED = 'attr:generated';
	const ATTR_DELETED = 'attr:deleted';
	const ATTR_UNCHANGED = 'attr:unchanged';
	const ATTR_WHITELINES = 'attr:white';
	const ATTR_MOVEAWAY = 'attr:moveaway';

	const LINES_CONTEXT = 8;

	const WHITESPACE_SHOW_ALL = 'show-all';
	const WHITESPACE_IGNORE_TRAILING = 'ignore-trailing';

	// TODO: This is now "Ignore Most" in the UI.
	const WHITESPACE_IGNORE_ALL = 'ignore-all';

	const WHITESPACE_IGNORE_FORCE = 'ignore-force';

	public function setOldLines(array $lines) {
	$this->old = $lines;
	return $this;
	}

	public function setNewLines(array $lines) {
	$this->new = $lines;
	return $this;
	}

	public function setSpecialAttributes(array $attributes) {
	$this->specialAttributes = $attributes;
	return $this;
	}

	public function setIntraLineDiffs(array $diffs) {
	$this->intra = $diffs;
	return $this;
	}

	public function setVisibileLinesMask(array $mask) {
	$this->visible = $mask;
	return $this;
	}

	/**
	* Configure which Changeset comments added to the right side of the visible
	* diff will be attached to. The ID must be the ID of a real Differential
	* Changeset.
	*
	* The complexity here is that we may show an arbitrary side of an arbitrary
	* changeset as either the left or right part of a diff. This method allows
	* the left and right halves of the displayed diff to be correctly mapped to
	* storage changesets.
	*
	* @param id The Differential Changeset ID that comments added to the right
	* side of the visible diff should be attached to.
	* @param bool If true, attach new comments to the right side of the storage
	* changeset. Note that this may be false, if the left side of
	* some storage changeset is being shown as the right side of
	* a display diff.
	* @return this
	*/
	public function setRightSideCommentMapping($id, $is_new) {
	$this->rightSideChangesetID = $id;
	$this->rightSideAttachesToNewFile = $is_new;
	return $this;
	}

	/**
	* See setRightSideCommentMapping(), but this sets information for the left
	* side of the display diff.
	*/
	public function setLeftSideCommentMapping($id, $is_new) {
	$this->leftSideChangesetID = $id;
	$this->leftSideAttachesToNewFile = $is_new;
	return $this;
	}

	public function setOriginals(
	DifferentialChangeset $left,
	DifferentialChangeset $right) {

	$this->originalLeft = $left;
	$this->originalRight = $right;
	}

	public function diffOriginals() {
	$engine = new PhabricatorDifferenceEngine();
	$changeset = $engine->generateChangesetFromFileContent(
	implode('', mpull($this->originalLeft->getHunks(), 'getChanges')),
	implode('', mpull($this->originalRight->getHunks(), 'getChanges')));

	$parser = new DifferentialHunkParser();

	return $parser->parseHunksForHighlightMasks(
	$changeset->getHunks(),
	$this->originalLeft->getHunks(),
	$this->originalRight->getHunks());
	}

	/**
	* Set a key for identifying this changeset in the render cache. If set, the
	* parser will attempt to use the changeset render cache, which can improve
	* performance for frequently-viewed changesets.
	*
	* By default, there is no render cache key and parsers do not use the cache.
	* This is appropriate for rarely-viewed changesets.
	*
	* NOTE: Currently, this key must be a valid Differential Changeset ID.
	*
	* @param string Key for identifying this changeset in the render cache.
	* @return this
	*/
	public function setRenderCacheKey($key) {
	$this->renderCacheKey = $key;
	return $this;
	}

	private function getRenderCacheKey() {
	return $this->renderCacheKey;
	}

	public function setChangeset(DifferentialChangeset $changeset) {
	$this->changeset = $changeset;

	$this->setFilename($changeset->getFilename());

	return $this;
	}

	public function setWhitespaceMode($whitespace_mode) {
	$this->whitespaceMode = $whitespace_mode;
	return $this;
	}

	public function setRenderingReference($ref) {
	$this->renderingReference = $ref;
	return $this;
	}

	private function getRenderingReference() {
	return $this->renderingReference;
	}

	public function getChangeset() {
	return $this->changeset;
	}

	public function setFilename($filename) {
	$this->filename = $filename;
	return $this;
	}

	public function setHandles(array $handles) {
	assert_instances_of($handles, 'PhabricatorObjectHandle');
	$this->handles = $handles;
	return $this;
	}

	public function setMarkupEngine(PhabricatorMarkupEngine $engine) {
	$this->markupEngine = $engine;
	return $this;
	}

	public function setUser(PhabricatorUser $user) {
	$this->user = $user;
	return $this;
	}

	public function setCoverage($coverage) {
	$this->coverage = $coverage;
	return $this;
	}
	private function getCoverage() {
	return $this->coverage;
	}

	public function parseInlineComment(
	PhabricatorInlineCommentInterface $comment) {

	// Parse only comments which are actually visible.
	if ($this->isCommentVisibleOnRenderedDiff($comment)) {
	$this->comments[] = $comment;
	}
	return $this;
	}

	private function loadCache() {
	$render_cache_key = $this->getRenderCacheKey();
	if (!$render_cache_key) {
	return false;
	}

	$data = null;

	$changeset = new DifferentialChangeset();
	$conn_r = $changeset->establishConnection('r');
	$data = queryfx_one(
	$conn_r,
	'SELECT * FROM %T WHERE id = %d',
	$changeset->getTableName().'_parse_cache',
	$render_cache_key);

	if (!$data) {
	return false;
	}

	if ($data['cache'][0] == '{') {
	// This is likely an old-style JSON cache which we will not be able to
	// deserialize.
	return false;
	}

	$data = unserialize($data['cache']);
	if (!is_array($data) \|\| !$data) {
	return false;
	}

	foreach (self::getCacheableProperties() as $cache_key) {
	if (!array_key_exists($cache_key, $data)) {
	// If we're missing a cache key, assume we're looking at an old cache
	// and ignore it.
	return false;
	}
	}

	if ($data['cacheVersion'] !== self::CACHE_VERSION) {
	return false;
	}

	// Someone displays contents of a partially cached shielded file.
	if (!isset($data['newRender']) && (!$this->isTopLevel \|\| $this->comments)) {
	return false;
	}

	unset($data['cacheVersion'], $data['cacheHost']);
	$cache_prop = array_select_keys($data, self::getCacheableProperties());
	foreach ($cache_prop as $cache_key => $v) {
	$this->$cache_key = $v;
	}

	return true;
	}

	protected static function getCacheableProperties() {
	return array(
	'visible',
	'new',
	'old',
	'intra',
	'newRender',
	'oldRender',
	'specialAttributes',
	'hunkStartLines',
	'cacheVersion',
	'cacheHost',
	);
	}

	public function saveCache() {
	if ($this->highlightErrors) {
	return false;
	}

	$render_cache_key = $this->getRenderCacheKey();
	if (!$render_cache_key) {
	return false;
	}

	$cache = array();
	foreach (self::getCacheableProperties() as $cache_key) {
	switch ($cache_key) {
	case 'cacheVersion':
	$cache[$cache_key] = self::CACHE_VERSION;
	break;
	case 'cacheHost':
	$cache[$cache_key] = php_uname('n');
	break;
	default:
	$cache[$cache_key] = $this->$cache_key;
	break;
	}
	}
	$cache = serialize($cache);

	// We don't want to waste too much space by a single changeset.
	if (strlen($cache) > self::CACHE_MAX_SIZE) {
	return;
	}

	$changeset = new DifferentialChangeset();
	$conn_w = $changeset->establishConnection('w');

	$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
	try {
	queryfx(
	$conn_w,
	'INSERT INTO %T (id, cache, dateCreated) VALUES (%d, %B, %d)
	ON DUPLICATE KEY UPDATE cache = VALUES(cache)',
	DifferentialChangeset::TABLE_CACHE,
	$render_cache_key,
	$cache,
	time());
	} catch (AphrontQueryException $ex) {
	// Ignore these exceptions. A common cause is that the cache is
	// larger than 'max_allowed_packet', in which case we're better off
	// not writing it.

	// TODO: It would be nice to tailor this more narrowly.
	}
	unset($unguarded);
	}

	private function markGenerated($new_corpus_block = '') {
	$generated_guess = (strpos($new_corpus_block, '@'.'generated') !== false);

	if (!$generated_guess) {
	$generated_path_regexps = PhabricatorEnv::getEnvConfig(
	'differential.generated-paths');
	foreach ($generated_path_regexps as $regexp) {
	if (preg_match($regexp, $this->changeset->getFilename())) {
	$generated_guess = true;
	break;
	}
	}
	}

	$event = new PhabricatorEvent(
	PhabricatorEventType::TYPE_DIFFERENTIAL_WILLMARKGENERATED,
	array(
	'corpus' => $new_corpus_block,
	'is_generated' => $generated_guess,
	)
	);
	PhutilEventEngine::dispatchEvent($event);

	$generated = $event->getValue('is_generated');
	$this->specialAttributes[self::ATTR_GENERATED] = $generated;
	}

	public function isGenerated() {
	return idx($this->specialAttributes, self::ATTR_GENERATED, false);
	}

	public function isDeleted() {
	return idx($this->specialAttributes, self::ATTR_DELETED, false);
	}

	public function isUnchanged() {
	return idx($this->specialAttributes, self::ATTR_UNCHANGED, false);
	}

	public function isWhitespaceOnly() {
	return idx($this->specialAttributes, self::ATTR_WHITELINES, false);
	}

	public function isMoveAway() {
	return idx($this->specialAttributes, self::ATTR_MOVEAWAY, false);
	}

	private function applyIntraline(&$render, $intra, $corpus) {

	foreach ($render as $key => $text) {
	if (isset($intra[$key])) {
	$render[$key] = ArcanistDiffUtils::applyIntralineDiff(
	$text,
	$intra[$key]);
	}
	}
	}

	private function getHighlightFuture($corpus) {
	$language = $this->highlightAs;

	if (!$language) {
	$language = $this->highlightEngine->getLanguageFromFilename(
	$this->filename);
	}

	return $this->highlightEngine->getHighlightFuture(
	$language,
	$corpus);
	}

	protected function processHighlightedSource($data, $result) {

	$result_lines = phutil_split_lines($result);
	foreach ($data as $key => $info) {
	if (!$info) {
	unset($result_lines[$key]);
	}
	}
	return $result_lines;
	}

	private function tryCacheStuff() {
	$whitespace_mode = $this->whitespaceMode;
	switch ($whitespace_mode) {
	case self::WHITESPACE_SHOW_ALL:
	case self::WHITESPACE_IGNORE_TRAILING:
	case self::WHITESPACE_IGNORE_FORCE:
	break;
	default:
	$whitespace_mode = self::WHITESPACE_IGNORE_ALL;
	break;
	}

	$skip_cache = ($whitespace_mode != self::WHITESPACE_IGNORE_ALL);
	if ($this->disableCache) {
	$skip_cache = true;
	}

	if ($this->characterEncoding) {
	$skip_cache = true;
	}

	if ($this->highlightAs) {
	$skip_cache = true;
	}

	$this->whitespaceMode = $whitespace_mode;

	$changeset = $this->changeset;

	if ($changeset->getFileType() != DifferentialChangeType::FILE_TEXT &&
	$changeset->getFileType() != DifferentialChangeType::FILE_SYMLINK) {

	$this->markGenerated();

	} else {
	if ($skip_cache \|\| !$this->loadCache()) {
	$this->process();
	if (!$skip_cache) {
	$this->saveCache();
	}
	}
	}
	}

	private function process() {
	$whitespace_mode = $this->whitespaceMode;
	$changeset = $this->changeset;

	$ignore_all = (($whitespace_mode == self::WHITESPACE_IGNORE_ALL) \|\|
	($whitespace_mode == self::WHITESPACE_IGNORE_FORCE));

	$force_ignore = ($whitespace_mode == self::WHITESPACE_IGNORE_FORCE);

	if (!$force_ignore) {
	if ($ignore_all && $changeset->getWhitespaceMatters()) {
	$ignore_all = false;
	}
	}

	// The "ignore all whitespace" algorithm depends on rediffing the
	// files, and we currently need complete representations of both
	// files to do anything reasonable. If we only have parts of the files,
	// don't use the "ignore all" algorithm.
	if ($ignore_all) {
	$hunks = $changeset->getHunks();
	if (count($hunks) !== 1) {
	$ignore_all = false;
	} else {
	$first_hunk = reset($hunks);
	if ($first_hunk->getOldOffset() != 1 \|\|
	$first_hunk->getNewOffset() != 1) {
	$ignore_all = false;
	}
	}
	}

	if ($ignore_all) {
	$old_file = $changeset->makeOldFile();
	$new_file = $changeset->makeNewFile();
	if ($old_file == $new_file) {
	// If the old and new files are exactly identical, the synthetic
	// diff below will give us nonsense and whitespace modes are
	// irrelevant anyway. This occurs when you, e.g., copy a file onto
	// itself in Subversion (see T271).
	$ignore_all = false;
	}
	}

	$hunk_parser = new DifferentialHunkParser();
	$hunk_parser->setWhitespaceMode($whitespace_mode);
	$hunk_parser->parseHunksForLineData($changeset->getHunks());

	// Depending on the whitespace mode, we may need to compute a different
	// set of changes than the set of changes in the hunk data (specificaly,
	// we might want to consider changed lines which have only whitespace
	// changes as unchanged).
	if ($ignore_all) {
	$engine = new PhabricatorDifferenceEngine();
	$engine->setIgnoreWhitespace(true);
	$no_whitespace_changeset = $engine->generateChangesetFromFileContent(
	$old_file,
	$new_file);

	$type_parser = new DifferentialHunkParser();
	$type_parser->parseHunksForLineData($no_whitespace_changeset->getHunks());

	$hunk_parser->setOldLineTypeMap($type_parser->getOldLineTypeMap());
	$hunk_parser->setNewLineTypeMap($type_parser->getNewLineTypeMap());
	}

	$hunk_parser->reparseHunksForSpecialAttributes();

	$unchanged = false;
	if (!$hunk_parser->getHasAnyChanges()) {
	$filetype = $this->changeset->getFileType();
	if ($filetype == DifferentialChangeType::FILE_TEXT \|\|
	$filetype == DifferentialChangeType::FILE_SYMLINK) {
	$unchanged = true;
	}
	}

	$moveaway = false;
	$changetype = $this->changeset->getChangeType();
	if ($changetype == DifferentialChangeType::TYPE_MOVE_AWAY) {
	// sometimes we show moved files as unchanged, sometimes deleted,
	// and sometimes inconsistent with what actually happened at the
	// destination of the move. Rather than make a false claim,
	// omit the 'not changed' notice if this is the source of a move
	$unchanged = false;
	$moveaway = true;
	}

	$this->setSpecialAttributes(array(
	self::ATTR_UNCHANGED => $unchanged,
	self::ATTR_DELETED => $hunk_parser->getIsDeleted(),
	self::ATTR_WHITELINES => !$hunk_parser->getHasTextChanges(),
	self::ATTR_MOVEAWAY => $moveaway,
	));

	$hunk_parser->generateIntraLineDiffs();
	$hunk_parser->generateVisibileLinesMask();

	$this->setOldLines($hunk_parser->getOldLines());
	$this->setNewLines($hunk_parser->getNewLines());
	$this->setIntraLineDiffs($hunk_parser->getIntraLineDiffs());
	$this->setVisibileLinesMask($hunk_parser->getVisibleLinesMask());
	$this->hunkStartLines = $hunk_parser->getHunkStartLines(
	$changeset->getHunks());

	$new_corpus = $hunk_parser->getNewCorpus();
	$new_corpus_block = implode('', $new_corpus);
	$this->markGenerated($new_corpus_block);

	if ($this->isTopLevel &&
	!$this->comments &&
	($this->isGenerated() \|\|
	$this->isUnchanged() \|\|
	$this->isDeleted())) {
	return;
	}

	$old_corpus = $hunk_parser->getOldCorpus();
	$old_corpus_block = implode('', $old_corpus);
	$old_future = $this->getHighlightFuture($old_corpus_block);
	$new_future = $this->getHighlightFuture($new_corpus_block);
	$futures = array(
	'old' => $old_future,
	'new' => $new_future,
	);
	$corpus_blocks = array(
	'old' => $old_corpus_block,
	'new' => $new_corpus_block,
	);

	$this->highlightErrors = false;
	- foreach (Futures($futures) as $key => $future) {
	+ foreach (new FutureIterator($futures) as $key => $future) {
	try {
	try {
	$highlighted = $future->resolve();
	} catch (PhutilSyntaxHighlighterException $ex) {
	$this->highlightErrors = true;
	$highlighted = id(new PhutilDefaultSyntaxHighlighter())
	->getHighlightFuture($corpus_blocks[$key])
	->resolve();
	}
	switch ($key) {
	case 'old':
	$this->oldRender = $this->processHighlightedSource(
	$this->old,
	$highlighted);
	break;
	case 'new':
	$this->newRender = $this->processHighlightedSource(
	$this->new,
	$highlighted);
	break;
	}
	} catch (Exception $ex) {
	phlog($ex);
	throw $ex;
	}
	}

	$this->applyIntraline(
	$this->oldRender,
	ipull($this->intra, 0),
	$old_corpus);
	$this->applyIntraline(
	$this->newRender,
	ipull($this->intra, 1),
	$new_corpus);
	}

	private function shouldRenderPropertyChangeHeader($changeset) {
	if (!$this->isTopLevel) {
	// We render properties only at top level; otherwise we get multiple
	// copies of them when a user clicks "Show More".
	return false;
	}

	return true;
	}

	public function render(
	$range_start = null,
	$range_len = null,
	$mask_force = array()) {

	// "Top level" renders are initial requests for the whole file, versus
	// requests for a specific range generated by clicking "show more". We
	// generate property changes and "shield" UI elements only for toplevel
	// requests.
	$this->isTopLevel = (($range_start === null) && ($range_len === null));
	$this->highlightEngine = PhabricatorSyntaxHighlighter::newEngine();

	$encoding = null;
	if ($this->characterEncoding) {
	// We are forcing this changeset to be interpreted with a specific
	// character encoding, so force all the hunks into that encoding and
	// propagate it to the renderer.
	$encoding = $this->characterEncoding;
	foreach ($this->changeset->getHunks() as $hunk) {
	$hunk->forceEncoding($this->characterEncoding);
	}
	} else {
	// We're just using the default, so tell the renderer what that is
	// (by reading the encoding from the first hunk).
	foreach ($this->changeset->getHunks() as $hunk) {
	$encoding = $hunk->getDataEncoding();
	break;
	}
	}

	$this->tryCacheStuff();
	$render_pch = $this->shouldRenderPropertyChangeHeader($this->changeset);

	$rows = max(
	count($this->old),
	count($this->new));

	$renderer = $this->getRenderer()
	->setChangeset($this->changeset)
	->setRenderPropertyChangeHeader($render_pch)
	->setIsTopLevel($this->isTopLevel)
	->setOldRender($this->oldRender)
	->setNewRender($this->newRender)
	->setHunkStartLines($this->hunkStartLines)
	->setOldChangesetID($this->leftSideChangesetID)
	->setNewChangesetID($this->rightSideChangesetID)
	->setOldAttachesToNewFile($this->leftSideAttachesToNewFile)
	->setNewAttachesToNewFile($this->rightSideAttachesToNewFile)
	->setCodeCoverage($this->getCoverage())
	->setRenderingReference($this->getRenderingReference())
	->setMarkupEngine($this->markupEngine)
	->setHandles($this->handles)
	->setOldLines($this->old)
	->setNewLines($this->new)
	->setOriginalCharacterEncoding($encoding);

	if ($this->user) {
	$renderer->setUser($this->user);
	}

	$shield = null;
	if ($this->isTopLevel && !$this->comments) {
	if ($this->isGenerated()) {
	$shield = $renderer->renderShield(
	pht(
	'This file contains generated code, which does not normally '.
	'need to be reviewed.'));
	} else if ($this->isUnchanged()) {
	$type = 'text';
	if (!$rows) {
	// NOTE: Normally, diffs which don't change files do not include
	// file content (for example, if you "chmod +x" a file and then
	// run "git show", the file content is not available). Similarly,
	// if you move a file from A to B without changing it, diffs normally
	// do not show the file content. In some cases `arc` is able to
	// synthetically generate content for these diffs, but for raw diffs
	// we'll never have it so we need to be prepared to not render a link.
	$type = 'none';
	}
	$shield = $renderer->renderShield(
	pht('The contents of this file were not changed.'),
	$type);
	} else if ($this->isMoveAway()) {
	$shield = null;
	} else if ($this->isWhitespaceOnly()) {
	$shield = $renderer->renderShield(
	pht('This file was changed only by adding or removing whitespace.'),
	'whitespace');
	} else if ($this->isDeleted()) {
	$shield = $renderer->renderShield(
	pht('This file was completely deleted.'));
	} else if ($this->changeset->getAffectedLineCount() > 2500) {
	$lines = number_format($this->changeset->getAffectedLineCount());
	$shield = $renderer->renderShield(
	pht(
	'This file has a very large number of changes (%s lines).',
	$lines));
	}
	}

	if ($shield) {
	return $renderer->renderChangesetTable($shield);
	}

	$old_comments = array();
	$new_comments = array();
	$old_mask = array();
	$new_mask = array();
	$feedback_mask = array();

	if ($this->comments) {
	foreach ($this->comments as $comment) {
	$start = max($comment->getLineNumber() - self::LINES_CONTEXT, 0);
	$end = $comment->getLineNumber() +
	$comment->getLineLength() +
	self::LINES_CONTEXT;
	$new_side = $this->isCommentOnRightSideWhenDisplayed($comment);
	for ($ii = $start; $ii <= $end; $ii++) {
	if ($new_side) {
	$new_mask[$ii] = true;
	} else {
	$old_mask[$ii] = true;
	}
	}
	}

	foreach ($this->old as $ii => $old) {
	if (isset($old['line']) && isset($old_mask[$old['line']])) {
	$feedback_mask[$ii] = true;
	}
	}

	foreach ($this->new as $ii => $new) {
	if (isset($new['line']) && isset($new_mask[$new['line']])) {
	$feedback_mask[$ii] = true;
	}
	}
	$this->comments = msort($this->comments, 'getID');
	foreach ($this->comments as $comment) {
	$final = $comment->getLineNumber() +
	$comment->getLineLength();
	$final = max(1, $final);
	if ($this->isCommentOnRightSideWhenDisplayed($comment)) {
	$new_comments[$final][] = $comment;
	} else {
	$old_comments[$final][] = $comment;
	}
	}
	}
	$renderer
	->setOldComments($old_comments)
	->setNewComments($new_comments);

	switch ($this->changeset->getFileType()) {
	case DifferentialChangeType::FILE_IMAGE:
	$old = null;
	$new = null;
	// TODO: Improve the architectural issue as discussed in D955
	// https://secure.phabricator.com/D955
	$reference = $this->getRenderingReference();
	$parts = explode('/', $reference);
	if (count($parts) == 2) {
	list($id, $vs) = $parts;
	} else {
	$id = $parts[0];
	$vs = 0;
	}
	$id = (int)$id;
	$vs = (int)$vs;

	if (!$vs) {
	$metadata = $this->changeset->getMetadata();
	$data = idx($metadata, 'attachment-data');

	$old_phid = idx($metadata, 'old:binary-phid');
	$new_phid = idx($metadata, 'new:binary-phid');
	} else {
	$vs_changeset = id(new DifferentialChangeset())->load($vs);
	$old_phid = null;
	$new_phid = null;

	// TODO: This is spooky, see D6851
	if ($vs_changeset) {
	$vs_metadata = $vs_changeset->getMetadata();
	$old_phid = idx($vs_metadata, 'new:binary-phid');
	}

	$changeset = id(new DifferentialChangeset())->load($id);
	if ($changeset) {
	$metadata = $changeset->getMetadata();
	$new_phid = idx($metadata, 'new:binary-phid');
	}
	}

	if ($old_phid \|\| $new_phid) {
	// grab the files, (micro) optimization for 1 query not 2
	$file_phids = array();
	if ($old_phid) {
	$file_phids[] = $old_phid;
	}
	if ($new_phid) {
	$file_phids[] = $new_phid;
	}

	// TODO: (T603) Probably fine to use omnipotent viewer here?
	$files = id(new PhabricatorFile())->loadAllWhere(
	'phid IN (%Ls)',
	$file_phids);
	foreach ($files as $file) {
	if (empty($file)) {
	continue;
	}
	if ($file->getPHID() == $old_phid) {
	$old = $file;
	} else if ($file->getPHID() == $new_phid) {
	$new = $file;
	}
	}
	}

	$renderer->attachOldFile($old);
	$renderer->attachNewFile($new);

	return $renderer->renderFileChange($old, $new, $id, $vs);
	case DifferentialChangeType::FILE_DIRECTORY:
	case DifferentialChangeType::FILE_BINARY:
	$output = $renderer->renderChangesetTable(null);
	return $output;
	}

	if ($this->originalLeft && $this->originalRight) {
	list($highlight_old, $highlight_new) = $this->diffOriginals();
	$highlight_old = array_flip($highlight_old);
	$highlight_new = array_flip($highlight_new);
	$renderer
	->setHighlightOld($highlight_old)
	->setHighlightNew($highlight_new);
	}
	$renderer
	->setOriginalOld($this->originalLeft)
	->setOriginalNew($this->originalRight);

	if ($range_start === null) {
	$range_start = 0;
	}
	if ($range_len === null) {
	$range_len = $rows;
	}
	$range_len = min($range_len, $rows - $range_start);

	list($gaps, $mask, $depths) = $this->calculateGapsMaskAndDepths(
	$mask_force,
	$feedback_mask,
	$range_start,
	$range_len);

	$renderer
	->setGaps($gaps)
	->setMask($mask)
	->setDepths($depths);

	$html = $renderer->renderTextChange(
	$range_start,
	$range_len,
	$rows);

	return $renderer->renderChangesetTable($html);
	}

	/**
	* This function calculates a lot of stuff we need to know to display
	* the diff:
	*
	* Gaps - compute gaps in the visible display diff, where we will render
	* "Show more context" spacers. If a gap is smaller than the context size,
	* we just display it. Otherwise, we record it into $gaps and will render a
	* "show more context" element instead of diff text below. A given $gap
	* is a tuple of $gap_line_number_start and $gap_length.
	*
	* Mask - compute the actual lines that need to be shown (because they
	* are near changes lines, near inline comments, or the request has
	* explicitly asked for them, i.e. resulting from the user clicking
	* "show more"). The $mask returned is a sparesely populated dictionary
	* of $visible_line_number => true.
	*
	* Depths - compute how indented any given line is. The $depths returned
	* is a sparesely populated dictionary of $visible_line_number => $depth.
	*
	* This function also has the side effect of modifying member variable
	* new such that tabs are normalized to spaces for each line of the diff.
	*
	* @return array($gaps, $mask, $depths)
	*/
	private function calculateGapsMaskAndDepths($mask_force,
	$feedback_mask,
	$range_start,
	$range_len) {

	// Calculate gaps and mask first
	$gaps = array();
	$gap_start = 0;
	$in_gap = false;
	$base_mask = $this->visible + $mask_force + $feedback_mask;
	$base_mask[$range_start + $range_len] = true;
	for ($ii = $range_start; $ii <= $range_start + $range_len; $ii++) {
	if (isset($base_mask[$ii])) {
	if ($in_gap) {
	$gap_length = $ii - $gap_start;
	if ($gap_length <= self::LINES_CONTEXT) {
	for ($jj = $gap_start; $jj <= $gap_start + $gap_length; $jj++) {
	$base_mask[$jj] = true;
	}
	} else {
	$gaps[] = array($gap_start, $gap_length);
	}
	$in_gap = false;
	}
	} else {
	if (!$in_gap) {
	$gap_start = $ii;
	$in_gap = true;
	}
	}
	}
	$gaps = array_reverse($gaps);
	$mask = $base_mask;

	// Time to calculate depth.
	// We need to go backwards to properly indent whitespace in this code:
	//
	// 0: class C {
	// 1:
	// 1: function f() {
	// 2:
	// 2: return;
	// 1:
	// 1: }
	// 0:
	// 0: }
	//
	$depths = array();
	$last_depth = 0;
	$range_end = $range_start + $range_len;
	if (!isset($this->new[$range_end])) {
	$range_end--;
	}
	for ($ii = $range_end; $ii >= $range_start; $ii--) {
	// We need to expand tabs to process mixed indenting and to round
	// correctly later.
	$line = str_replace("\t", ' ', $this->new[$ii]['text']);
	$trimmed = ltrim($line);
	if ($trimmed != '') {
	// We round down to flatten "/*" and " ".
	$last_depth = floor((strlen($line) - strlen($trimmed)) / 2);
	}
	$depths[$ii] = $last_depth;
	}

	return array($gaps, $mask, $depths);
	}

	/**
	* Determine if an inline comment will appear on the rendered diff,
	* taking into consideration which halves of which changesets will actually
	* be shown.
	*
	* @param PhabricatorInlineCommentInterface Comment to test for visibility.
	* @return bool True if the comment is visible on the rendered diff.
	*/
	private function isCommentVisibleOnRenderedDiff(
	PhabricatorInlineCommentInterface $comment) {

	$changeset_id = $comment->getChangesetID();
	$is_new = $comment->getIsNewFile();

	if ($changeset_id == $this->rightSideChangesetID &&
	$is_new == $this->rightSideAttachesToNewFile) {
	return true;
	}

	if ($changeset_id == $this->leftSideChangesetID &&
	$is_new == $this->leftSideAttachesToNewFile) {
	return true;
	}

	return false;
	}


	/**
	* Determine if a comment will appear on the right side of the display diff.
	* Note that the comment must appear somewhere on the rendered changeset, as
	* per isCommentVisibleOnRenderedDiff().
	*
	* @param PhabricatorInlineCommentInterface Comment to test for display
	* location.
	* @return bool True for right, false for left.
	*/
	private function isCommentOnRightSideWhenDisplayed(
	PhabricatorInlineCommentInterface $comment) {

	if (!$this->isCommentVisibleOnRenderedDiff($comment)) {
	throw new Exception('Comment is not visible on changeset!');
	}

	$changeset_id = $comment->getChangesetID();
	$is_new = $comment->getIsNewFile();

	if ($changeset_id == $this->rightSideChangesetID &&
	$is_new == $this->rightSideAttachesToNewFile) {
	return true;
	}

	return false;
	}

	/**
	* Parse the 'range' specification that this class and the client-side JS
	* emit to indicate that a user clicked "Show more..." on a diff. Generally,
	* use is something like this:
	*
	* $spec = $request->getStr('range');
	* $parsed = DifferentialChangesetParser::parseRangeSpecification($spec);
	* list($start, $end, $mask) = $parsed;
	* $parser->render($start, $end, $mask);
	*
	* @param string Range specification, indicating the range of the diff that
	* should be rendered.
	* @return tuple List of <start, end, mask> suitable for passing to
	* @{method:render}.
	*/
	public static function parseRangeSpecification($spec) {
	$range_s = null;
	$range_e = null;
	$mask = array();

	if ($spec) {
	$match = null;
	if (preg_match('@^(\d+)-(\d+)(?:/(\d+)-(\d+))?$@', $spec, $match)) {
	$range_s = (int)$match[1];
	$range_e = (int)$match[2];
	if (count($match) > 3) {
	$start = (int)$match[3];
	$len = (int)$match[4];
	for ($ii = $start; $ii < $start + $len; $ii++) {
	$mask[$ii] = true;
	}
	}
	}
	}

	return array($range_s, $range_e, $mask);
	}

	/**
	* Render "modified coverage" information; test coverage on modified lines.
	* This synthesizes diff information with unit test information into a useful
	* indicator of how well tested a change is.
	*/
	public function renderModifiedCoverage() {
	$na = phutil_tag('em', array(), '-');

	$coverage = $this->getCoverage();
	if (!$coverage) {
	return $na;
	}

	$covered = 0;
	$not_covered = 0;

	foreach ($this->new as $k => $new) {
	if (!$new['line']) {
	continue;
	}

	if (!$new['type']) {
	continue;
	}

	if (empty($coverage[$new['line'] - 1])) {
	continue;
	}

	switch ($coverage[$new['line'] - 1]) {
	case 'C':
	$covered++;
	break;
	case 'U':
	$not_covered++;
	break;
	}
	}

	if (!$covered && !$not_covered) {
	return $na;
	}

	return sprintf('%d%%', 100 * ($covered / ($covered + $not_covered)));
	}

	public function detectCopiedCode(
	array $changesets,
	$min_width = 30,
	$min_lines = 3) {

	assert_instances_of($changesets, 'DifferentialChangeset');

	$map = array();
	$files = array();
	$types = array();
	foreach ($changesets as $changeset) {
	$file = $changeset->getFilename();
	foreach ($changeset->getHunks() as $hunk) {
	$line = $hunk->getOldOffset();
	foreach (explode("\n", $hunk->getChanges()) as $code) {
	$type = (isset($code[0]) ? $code[0] : '');
	if ($type == '-' \|\| $type == ' ') {
	$code = trim(substr($code, 1));
	$files[$file][$line] = $code;
	$types[$file][$line] = $type;
	if (strlen($code) >= $min_width) {
	$map[$code][] = array($file, $line);
	}
	$line++;
	}
	}
	}
	}

	foreach ($changesets as $changeset) {
	$copies = array();
	foreach ($changeset->getHunks() as $hunk) {
	$added = array_map('trim', $hunk->getAddedLines());
	for (reset($added); list($line, $code) = each($added); ) {
	if (isset($map[$code])) { // We found a long matching line.

	if (count($map[$code]) > 16) {
	// If there are a large number of identical lines in this diff,
	// don't try to figure out where this block came from: the
	// analysis is O(N^2), since we need to compare every line
	// against every other line. Even if we arrive at a result, it
	// is unlikely to be meaningful. See T5041.
	continue 2;
	}

	$best_length = 0;
	foreach ($map[$code] as $val) { // Explore all candidates.
	list($file, $orig_line) = $val;
	$length = 1;
	// Search also backwards for short lines.
	foreach (array(-1, 1) as $direction) {
	$offset = $direction;
	while (!isset($copies[$line + $offset]) &&
	isset($added[$line + $offset]) &&
	idx($files[$file], $orig_line + $offset) ===
	$added[$line + $offset]) {
	$length++;
	$offset += $direction;
	}
	}
	if ($length > $best_length \|\|
	($length == $best_length && // Prefer moves.
	idx($types[$file], $orig_line) == '-')) {
	$best_length = $length;
	// ($offset - 1) contains number of forward matching lines.
	$best_offset = $offset - 1;
	$best_file = $file;
	$best_line = $orig_line;
	}
	}
	$file = ($best_file == $changeset->getFilename() ? '' : $best_file);
	for ($i = $best_length; $i--; ) {
	$type = idx($types[$best_file], $best_line + $best_offset - $i);
	$copies[$line + $best_offset - $i] = ($best_length < $min_lines
	? array() // Ignore short blocks.
	: array($file, $best_line + $best_offset - $i, $type));
	}
	for ($i = 0; $i < $best_offset; $i++) {
	next($added);
	}
	}
	}
	}
	$copies = array_filter($copies);
	if ($copies) {
	$metadata = $changeset->getMetadata();
	$metadata['copy:lines'] = $copies;
	$changeset->setMetadata($metadata);
	}
	}
	return $changesets;
	}

	}
	diff --git a/src/applications/diffusion/DiffusionLintSaveRunner.php b/src/applications/diffusion/DiffusionLintSaveRunner.php
	index 23bc70a2ef..6adcde8dda 100644
	--- a/src/applications/diffusion/DiffusionLintSaveRunner.php
	+++ b/src/applications/diffusion/DiffusionLintSaveRunner.php
	@@ -1,289 +1,291 @@
	<?php

	final class DiffusionLintSaveRunner {
	private $arc = 'arc';
	private $severity = ArcanistLintSeverity::SEVERITY_ADVICE;
	private $all = false;
	private $chunkSize = 256;
	private $needsBlame = false;

	private $svnRoot;
	private $lintCommit;
	private $branch;
	private $conn;
	private $deletes = array();
	private $inserts = array();
	private $blame = array();


	public function setArc($path) {
	$this->arc = $path;
	return $this;
	}

	public function setSeverity($string) {
	$this->severity = $string;
	return $this;
	}

	public function setAll($bool) {
	$this->all = $bool;
	return $this;
	}

	public function setChunkSize($number) {
	$this->chunkSize = $number;
	return $this;
	}

	public function setNeedsBlame($boolean) {
	$this->needsBlame = $boolean;
	return $this;
	}


	public function run($dir) {
	$working_copy = ArcanistWorkingCopyIdentity::newFromPath($dir);
	$configuration_manager = new ArcanistConfigurationManager();
	$configuration_manager->setWorkingCopyIdentity($working_copy);
	$api = ArcanistRepositoryAPI::newAPIFromConfigurationManager(
	$configuration_manager);

	$this->svnRoot = id(new PhutilURI($api->getSourceControlPath()))->getPath();
	if ($api instanceof ArcanistGitAPI) {
	$svn_fetch = $api->getGitConfig('svn-remote.svn.fetch');
	list($this->svnRoot) = explode(':', $svn_fetch);
	if ($this->svnRoot != '') {
	$this->svnRoot = '/'.$this->svnRoot;
	}
	}

	$project_id = $working_copy->getProjectID();
	$project = id(new PhabricatorRepositoryArcanistProject())
	->loadOneWhere('name = %s', $project_id);
	if (!$project \|\| !$project->getRepositoryID()) {
	throw new Exception("Couldn't find repository for {$project_id}.");
	}

	$branch_name = $api->getBranchName();

	$this->branch = PhabricatorRepositoryBranch::loadOrCreateBranch(
	$project->getRepositoryID(),
	$branch_name);
	$this->conn = $this->branch->establishConnection('w');

	$this->lintCommit = null;
	if (!$this->all) {
	$this->lintCommit = $this->branch->getLintCommit();
	}

	if ($this->lintCommit) {
	try {
	$commit = $this->lintCommit;
	if ($this->svnRoot) {
	$commit = $api->getCanonicalRevisionName('@'.$commit);
	}
	$all_files = $api->getChangedFiles($commit);
	} catch (ArcanistCapabilityNotSupportedException $ex) {
	$this->lintCommit = null;
	}
	}


	if (!$this->lintCommit) {
	$where = ($this->svnRoot
	? qsprintf($this->conn, 'AND path LIKE %>', $this->svnRoot.'/')
	: '');
	queryfx(
	$this->conn,
	'DELETE FROM %T WHERE branchID = %d %Q',
	PhabricatorRepository::TABLE_LINTMESSAGE,
	$this->branch->getID(),
	$where);
	$all_files = $api->getAllFiles();
	}

	$count = 0;

	$files = array();
	foreach ($all_files as $file => $val) {
	$count++;
	if (!$this->lintCommit) {
	$file = $val;
	} else {
	$this->deletes[] = $this->svnRoot.'/'.$file;
	if ($val & ArcanistRepositoryAPI::FLAG_DELETED) {
	continue;
	}
	}
	$files[$file] = $file;

	if (count($files) >= $this->chunkSize) {
	$this->runArcLint($files);
	$files = array();
	}
	}

	$this->runArcLint($files);
	$this->saveLintMessages();

	$this->lintCommit = $api->getUnderlyingWorkingCopyRevision();
	$this->branch->setLintCommit($this->lintCommit);
	$this->branch->save();

	if ($this->blame) {
	$this->blameAuthors();
	$this->blame = array();
	}

	return $count;
	}


	private function runArcLint(array $files) {
	if (!$files) {
	return;
	}

	echo '.';
	try {
	$future = new ExecFuture(
	'%C lint --severity %s --output json %Ls',
	$this->arc,
	$this->severity,
	$files);

	foreach (new LinesOfALargeExecFuture($future) as $json) {
	$paths = json_decode($json, true);
	if (!is_array($paths)) {
	fprintf(STDERR, "Invalid JSON: {$json}\n");
	continue;
	}

	foreach ($paths as $path => $messages) {
	if (!isset($files[$path])) {
	continue;
	}

	foreach ($messages as $message) {
	$line = idx($message, 'line', 0);

	$this->inserts[] = qsprintf(
	$this->conn,
	'(%d, %s, %d, %s, %s, %s, %s)',
	$this->branch->getID(),
	$this->svnRoot.'/'.$path,
	$line,
	idx($message, 'code', ''),
	idx($message, 'severity', ''),
	idx($message, 'name', ''),
	idx($message, 'description', ''));

	if ($line && $this->needsBlame) {
	$this->blame[$path][$line] = true;
	}
	}

	if (count($this->deletes) >= 1024 \|\| count($this->inserts) >= 256) {
	$this->saveLintMessages();
	}
	}
	}

	} catch (Exception $ex) {
	fprintf(STDERR, $ex->getMessage()."\n");
	}
	}


	private function saveLintMessages() {
	$this->conn->openTransaction();

	foreach (array_chunk($this->deletes, 1024) as $paths) {
	queryfx(
	$this->conn,
	'DELETE FROM %T WHERE branchID = %d AND path IN (%Ls)',
	PhabricatorRepository::TABLE_LINTMESSAGE,
	$this->branch->getID(),
	$paths);
	}

	foreach (array_chunk($this->inserts, 256) as $values) {
	queryfx(
	$this->conn,
	'INSERT INTO %T
	(branchID, path, line, code, severity, name, description)
	VALUES %Q',
	PhabricatorRepository::TABLE_LINTMESSAGE,
	implode(', ', $values));
	}

	$this->conn->saveTransaction();

	$this->deletes = array();
	$this->inserts = array();
	}


	private function blameAuthors() {
	$repository = id(new PhabricatorRepositoryQuery())
	->setViewer(PhabricatorUser::getOmnipotentUser())
	->withIDs(array($this->branch->getRepositoryID()))
	->executeOne();

	$queries = array();
	$futures = array();
	foreach ($this->blame as $path => $lines) {
	$drequest = DiffusionRequest::newFromDictionary(array(
	'user' => PhabricatorUser::getOmnipotentUser(),
	'initFromConduit' => false,
	'repository' => $repository,
	'branch' => $this->branch->getName(),
	'path' => $path,
	'commit' => $this->lintCommit,
	));
	$query = DiffusionFileContentQuery::newFromDiffusionRequest($drequest)
	->setNeedsBlame(true);
	$queries[$path] = $query;
	$futures[$path] = $query->getFileContentFuture();
	}

	$authors = array();

	- foreach (Futures($futures)->limit(8) as $path => $future) {
	+ $futures = id(new FutureIterator($futures))
	+ ->limit(8);
	+ foreach ($futures as $path => $future) {
	$queries[$path]->loadFileContentFromFuture($future);
	list(, $rev_list, $blame_dict) = $queries[$path]->getBlameData();
	foreach (array_keys($this->blame[$path]) as $line) {
	$commit_identifier = $rev_list[$line - 1];
	$author = idx($blame_dict[$commit_identifier], 'authorPHID');
	if ($author) {
	$authors[$author][$path][] = $line;
	}
	}
	}

	if ($authors) {
	$this->conn->openTransaction();

	foreach ($authors as $author => $paths) {
	$where = array();
	foreach ($paths as $path => $lines) {
	$where[] = qsprintf(
	$this->conn,
	'(path = %s AND line IN (%Ld))',
	$this->svnRoot.'/'.$path,
	$lines);
	}
	queryfx(
	$this->conn,
	'UPDATE %T SET authorPHID = %s WHERE %Q',
	PhabricatorRepository::TABLE_LINTMESSAGE,
	$author,
	implode(' OR ', $where));
	}

	$this->conn->saveTransaction();
	}
	}

	}
	diff --git a/src/applications/diffusion/conduit/DiffusionDiffQueryConduitAPIMethod.php b/src/applications/diffusion/conduit/DiffusionDiffQueryConduitAPIMethod.php
	index c0b311c8eb..4dc2a2bac9 100644
	--- a/src/applications/diffusion/conduit/DiffusionDiffQueryConduitAPIMethod.php
	+++ b/src/applications/diffusion/conduit/DiffusionDiffQueryConduitAPIMethod.php
	@@ -1,239 +1,239 @@
	<?php

	final class DiffusionDiffQueryConduitAPIMethod
	extends DiffusionQueryConduitAPIMethod {

	private $effectiveCommit;

	public function getAPIMethodName() {
	return 'diffusion.diffquery';
	}

	public function getMethodDescription() {
	return
	'Get diff information from a repository for a specific path at an '.
	'(optional) commit.';
	}

	public function defineReturnType() {
	return 'array';
	}

	protected function defineCustomParamTypes() {
	return array(
	'path' => 'required string',
	'commit' => 'optional string',
	);
	}

	protected function getResult(ConduitAPIRequest $request) {
	$result = parent::getResult($request);

	return array(
	'changes' => mpull($result, 'toDictionary'),
	'effectiveCommit' => $this->getEffectiveCommit($request),
	);
	}

	protected function getGitResult(ConduitAPIRequest $request) {
	return $this->getGitOrMercurialResult($request);
	}

	protected function getMercurialResult(ConduitAPIRequest $request) {
	return $this->getGitOrMercurialResult($request);
	}

	/**
	* NOTE: We have to work particularly hard for SVN as compared to other VCS.
	* That's okay but means this shares little code with the other VCS.
	*/
	protected function getSVNResult(ConduitAPIRequest $request) {
	$drequest = $this->getDiffusionRequest();
	$repository = $drequest->getRepository();

	$effective_commit = $this->getEffectiveCommit($request);
	if (!$effective_commit) {
	return $this->getEmptyResult();
	}

	$drequest = clone $drequest;
	$drequest->updateSymbolicCommit($effective_commit);

	$path_change_query = DiffusionPathChangeQuery::newFromDiffusionRequest(
	$drequest);
	$path_changes = $path_change_query->loadChanges();

	$path = null;
	foreach ($path_changes as $change) {
	if ($change->getPath() == $drequest->getPath()) {
	$path = $change;
	}
	}

	if (!$path) {
	return $this->getEmptyResult();
	}

	$change_type = $path->getChangeType();
	switch ($change_type) {
	case DifferentialChangeType::TYPE_MULTICOPY:
	case DifferentialChangeType::TYPE_DELETE:
	if ($path->getTargetPath()) {
	$old = array(
	$path->getTargetPath(),
	$path->getTargetCommitIdentifier(),
	);
	} else {
	$old = array($path->getPath(), $path->getCommitIdentifier() - 1);
	}
	$old_name = $path->getPath();
	$new_name = '';
	$new = null;
	break;
	case DifferentialChangeType::TYPE_ADD:
	$old = null;
	$new = array($path->getPath(), $path->getCommitIdentifier());
	$old_name = '';
	$new_name = $path->getPath();
	break;
	case DifferentialChangeType::TYPE_MOVE_HERE:
	case DifferentialChangeType::TYPE_COPY_HERE:
	$old = array(
	$path->getTargetPath(),
	$path->getTargetCommitIdentifier(),
	);
	$new = array($path->getPath(), $path->getCommitIdentifier());
	$old_name = $path->getTargetPath();
	$new_name = $path->getPath();
	break;
	case DifferentialChangeType::TYPE_MOVE_AWAY:
	$old = array(
	$path->getPath(),
	$path->getCommitIdentifier() - 1,
	);
	$old_name = $path->getPath();
	$new_name = null;
	$new = null;
	break;
	default:
	$old = array($path->getPath(), $path->getCommitIdentifier() - 1);
	$new = array($path->getPath(), $path->getCommitIdentifier());
	$old_name = $path->getPath();
	$new_name = $path->getPath();
	break;
	}

	$futures = array(
	'old' => $this->buildSVNContentFuture($old),
	'new' => $this->buildSVNContentFuture($new),
	);
	$futures = array_filter($futures);

	- foreach (Futures($futures) as $key => $future) {
	+ foreach (new FutureIterator($futures) as $key => $future) {
	$stdout = '';
	try {
	list($stdout) = $future->resolvex();
	} catch (CommandException $e) {
	if ($path->getFileType() != DifferentialChangeType::FILE_DIRECTORY) {
	throw $e;
	}
	}
	$futures[$key] = $stdout;
	}

	$old_data = idx($futures, 'old', '');
	$new_data = idx($futures, 'new', '');

	$engine = new PhabricatorDifferenceEngine();
	$engine->setOldName($old_name);
	$engine->setNewName($new_name);
	$raw_diff = $engine->generateRawDiffFromFileContent($old_data, $new_data);

	$arcanist_changes = DiffusionPathChange::convertToArcanistChanges(
	$path_changes);

	$parser = $this->getDefaultParser();
	$parser->setChanges($arcanist_changes);
	$parser->forcePath($path->getPath());
	$changes = $parser->parseDiff($raw_diff);

	$change = $changes[$path->getPath()];

	return array($change);
	}

	private function getEffectiveCommit(ConduitAPIRequest $request) {
	if ($this->effectiveCommit === null) {
	$drequest = $this->getDiffusionRequest();

	$path = $drequest->getPath();
	$result = DiffusionQuery::callConduitWithDiffusionRequest(
	$request->getUser(),
	$drequest,
	'diffusion.lastmodifiedquery',
	array(
	'paths' => array($path => $drequest->getStableCommit()),
	));

	$this->effectiveCommit = idx($result, $path);
	}

	return $this->effectiveCommit;
	}

	private function buildSVNContentFuture($spec) {
	if (!$spec) {
	return null;
	}

	$drequest = $this->getDiffusionRequest();
	$repository = $drequest->getRepository();

	list($ref, $rev) = $spec;
	return $repository->getRemoteCommandFuture(
	'cat %s',
	$repository->getSubversionPathURI($ref, $rev));
	}

	private function getGitOrMercurialResult(ConduitAPIRequest $request) {
	$drequest = $this->getDiffusionRequest();
	$repository = $drequest->getRepository();

	$effective_commit = $this->getEffectiveCommit($request);
	if (!$effective_commit) {
	return $this->getEmptyResult(1);
	}

	$raw_query = DiffusionRawDiffQuery::newFromDiffusionRequest($drequest)
	->setAnchorCommit($effective_commit);

	$raw_diff = $raw_query->loadRawDiff();
	if (!$raw_diff) {
	return $this->getEmptyResult(2);
	}

	$parser = $this->getDefaultParser();
	$changes = $parser->parseDiff($raw_diff);

	return $changes;
	}

	private function getDefaultParser() {
	$drequest = $this->getDiffusionRequest();
	$repository = $drequest->getRepository();

	$parser = new ArcanistDiffParser();
	$try_encoding = $repository->getDetail('encoding');
	if ($try_encoding) {
	$parser->setTryEncoding($try_encoding);
	}
	$parser->setDetectBinaryFiles(true);

	return $parser;
	}

	private function getEmptyResult() {
	return array();
	}

	}
	diff --git a/src/applications/diffusion/conduit/DiffusionTagsQueryConduitAPIMethod.php b/src/applications/diffusion/conduit/DiffusionTagsQueryConduitAPIMethod.php
	index 4f8d6a959f..1ece4dd85b 100644
	--- a/src/applications/diffusion/conduit/DiffusionTagsQueryConduitAPIMethod.php
	+++ b/src/applications/diffusion/conduit/DiffusionTagsQueryConduitAPIMethod.php
	@@ -1,167 +1,168 @@
	<?php

	final class DiffusionTagsQueryConduitAPIMethod
	extends DiffusionQueryConduitAPIMethod {

	public function getAPIMethodName() {
	return 'diffusion.tagsquery';
	}

	public function getMethodDescription() {
	return pht('Retrieve information about tags in a repository.');
	}

	public function defineReturnType() {
	return 'array';
	}

	protected function defineCustomParamTypes() {
	return array(
	'names' => 'optional list<string>',
	'commit' => 'optional string',
	'needMessages' => 'optional bool',
	'offset' => 'optional int',
	'limit' => 'optional int',
	);
	}

	protected function getGitResult(ConduitAPIRequest $request) {
	$drequest = $this->getDiffusionRequest();
	$repository = $drequest->getRepository();
	$commit = $drequest->getSymbolicCommit();

	$commit_filter = null;
	if ($commit) {
	$commit_filter = $this->loadTagNamesForCommit($commit);
	}

	$name_filter = $request->getValue('names', null);

	$all_tags = $this->loadGitTagList();
	$all_tags = mpull($all_tags, null, 'getName');

	if ($name_filter !== null) {
	$all_tags = array_intersect_key($all_tags, array_fuse($name_filter));
	}
	if ($commit_filter !== null) {
	$all_tags = array_intersect_key($all_tags, $commit_filter);
	}

	$tags = array_values($all_tags);

	$offset = $request->getValue('offset');
	$limit = $request->getValue('limit');
	if ($offset) {
	$tags = array_slice($tags, $offset);
	}

	if ($limit) {
	$tags = array_slice($tags, 0, $limit);
	}

	if ($request->getValue('needMessages')) {
	$this->loadMessagesForTags($all_tags);
	}

	return mpull($tags, 'toDictionary');
	}

	private function loadGitTagList() {
	$drequest = $this->getDiffusionRequest();
	$repository = $drequest->getRepository();

	$refs = id(new DiffusionLowLevelGitRefQuery())
	->setRepository($repository)
	->withIsTag(true)
	->execute();

	$tags = array();
	foreach ($refs as $ref) {
	$fields = $ref->getRawFields();
	$tag = id(new DiffusionRepositoryTag())
	->setAuthor($fields['author'])
	->setEpoch($fields['epoch'])
	->setCommitIdentifier($ref->getCommitIdentifier())
	->setName($ref->getShortName())
	->setDescription($fields['subject'])
	->setType('git/'.$fields['objecttype']);

	$tags[] = $tag;
	}

	return $tags;
	}

	private function loadTagNamesForCommit($commit) {
	$drequest = $this->getDiffusionRequest();
	$repository = $drequest->getRepository();

	list($err, $stdout) = $repository->execLocalCommand(
	'tag -l --contains %s',
	$commit);

	if ($err) {
	// Git exits with an error code if the commit is bogus.
	return array();
	}

	$stdout = rtrim($stdout, "\n");
	if (!strlen($stdout)) {
	return array();
	}

	$tag_names = explode("\n", $stdout);
	$tag_names = array_fill_keys($tag_names, true);

	return $tag_names;
	}

	private function loadMessagesForTags(array $tags) {
	assert_instances_of($tags, 'DiffusionRepositoryTag');

	$drequest = $this->getDiffusionRequest();
	$repository = $drequest->getRepository();

	$futures = array();
	foreach ($tags as $key => $tag) {
	$futures[$key] = $repository->getLocalCommandFuture(
	'cat-file tag %s',
	$tag->getName());
	}

	- Futures($futures)->resolveAll();
	+ id(new FutureIterator($futures))
	+ ->resolveAll();

	foreach ($tags as $key => $tag) {
	$future = $futures[$key];
	list($err, $stdout) = $future->resolve();

	$message = null;
	if ($err) {
	// Not all tags are actually "tag" objects: a "tag" object is only
	// created if you provide a message or sign the tag. Tags created with
	// `git tag x [commit]` are "lightweight tags" and `git cat-file tag`
	// will fail on them. This is fine: they don't have messages.
	} else {
	$parts = explode("\n\n", $stdout, 2);
	if (count($parts) == 2) {
	$message = last($parts);
	}
	}

	$tag->attachMessage($message);
	}

	return $tags;
	}

	protected function getMercurialResult(ConduitAPIRequest $request) {
	// For now, we don't support Mercurial tags via API.
	return array();
	}

	protected function getSVNResult(ConduitAPIRequest $request) {
	// Subversion has no meaningful concept of tags.
	return array();
	}

	}
	diff --git a/src/applications/diffusion/controller/DiffusionBrowseSearchController.php b/src/applications/diffusion/controller/DiffusionBrowseSearchController.php
	index 16bb1bc761..8b6f91c38e 100644
	--- a/src/applications/diffusion/controller/DiffusionBrowseSearchController.php
	+++ b/src/applications/diffusion/controller/DiffusionBrowseSearchController.php
	@@ -1,206 +1,208 @@
	<?php

	final class DiffusionBrowseSearchController extends DiffusionBrowseController {

	public function processRequest() {
	$drequest = $this->diffusionRequest;

	$actions = $this->buildActionView($drequest);
	$properties = $this->buildPropertyView($drequest, $actions);

	$object_box = id(new PHUIObjectBoxView())
	->setHeader($this->buildHeaderView($drequest))
	->addPropertyList($properties);

	$content = array();

	$content[] = $object_box;
	$content[] = $this->renderSearchForm($collapsed = false);
	$content[] = $this->renderSearchResults();

	$crumbs = $this->buildCrumbs(
	array(
	'branch' => true,
	'path' => true,
	'view' => 'browse',
	));

	return $this->buildApplicationPage(
	array(
	$crumbs,
	$content,
	),
	array(
	'title' => array(
	nonempty(basename($drequest->getPath()), '/'),
	$drequest->getRepository()->getCallsign().' Repository',
	),
	));
	}

	private function renderSearchResults() {
	$drequest = $this->getDiffusionRequest();
	$repository = $drequest->getRepository();
	$results = array();

	$limit = 100;
	$page = $this->getRequest()->getInt('page', 0);
	$pager = new AphrontPagerView();
	$pager->setPageSize($limit);
	$pager->setOffset($page);
	$pager->setURI($this->getRequest()->getRequestURI(), 'page');

	$search_mode = null;

	try {
	if (strlen($this->getRequest()->getStr('grep'))) {
	$search_mode = 'grep';
	$query_string = $this->getRequest()->getStr('grep');
	$results = $this->callConduitWithDiffusionRequest(
	'diffusion.searchquery',
	array(
	'grep' => $query_string,
	'commit' => $drequest->getStableCommit(),
	'path' => $drequest->getPath(),
	'limit' => $limit + 1,
	'offset' => $page,
	));
	} else { // Filename search.
	$search_mode = 'find';
	$query_string = $this->getRequest()->getStr('find');
	$results = $this->callConduitWithDiffusionRequest(
	'diffusion.querypaths',
	array(
	'pattern' => $query_string,
	'commit' => $drequest->getStableCommit(),
	'path' => $drequest->getPath(),
	'limit' => $limit + 1,
	'offset' => $page,
	));
	}
	} catch (ConduitException $ex) {
	$err = $ex->getErrorDescription();
	if ($err != '') {
	return id(new AphrontErrorView())
	->setTitle(pht('Search Error'))
	->appendChild($err);
	}
	}

	$results = $pager->sliceResults($results);

	if ($search_mode == 'grep') {
	$table = $this->renderGrepResults($results);
	$header = pht(
	'File content matching "%s" under "%s"',
	$query_string,
	nonempty($drequest->getPath(), '/'));
	} else {
	$table = $this->renderFindResults($results);
	$header = pht(
	'Paths matching "%s" under "%s"',
	$query_string,
	nonempty($drequest->getPath(), '/'));
	}

	$box = id(new PHUIObjectBoxView())
	->setHeaderText($header)
	->appendChild($table);

	$pager_box = id(new PHUIBoxView())
	->addMargin(PHUI::MARGIN_LARGE)
	->appendChild($pager);

	return array($box, $pager_box);
	}

	private function renderGrepResults(array $results) {
	$drequest = $this->getDiffusionRequest();

	require_celerity_resource('syntax-highlighting-css');

	// NOTE: This can be wrong because we may find the string inside the
	// comment. But it's correct in most cases and highlighting the whole file
	// would be too expensive.
	$futures = array();
	$engine = PhabricatorSyntaxHighlighter::newEngine();
	foreach ($results as $result) {
	list($path, $line, $string) = $result;
	$futures["{$path}:{$line}"] = $engine->getHighlightFuture(
	$engine->getLanguageFromFilename($path),
	ltrim($string));
	}

	try {
	- Futures($futures)->limit(8)->resolveAll();
	+ id(new FutureIterator($futures))
	+ ->limit(8)
	+ ->resolveAll();
	} catch (PhutilSyntaxHighlighterException $ex) {}

	$rows = array();
	foreach ($results as $result) {
	list($path, $line, $string) = $result;

	$href = $drequest->generateURI(array(
	'action' => 'browse',
	'path' => $path,
	'line' => $line,
	));

	try {
	$string = $futures["{$path}:{$line}"]->resolve();
	} catch (PhutilSyntaxHighlighterException $ex) {}

	$string = phutil_tag(
	'pre',
	array('class' => 'PhabricatorMonospaced'),
	$string);

	$path = Filesystem::readablePath($path, $drequest->getPath());

	$rows[] = array(
	phutil_tag('a', array('href' => $href), $path),
	$line,
	$string,
	);
	}

	$table = id(new AphrontTableView($rows))
	->setClassName('remarkup-code')
	->setHeaders(array(pht('Path'), pht('Line'), pht('String')))
	->setColumnClasses(array('', 'n', 'wide'))
	->setNoDataString(
	pht(
	'The pattern you searched for was not found in the content of any '.
	'files.'));

	return $table;
	}

	private function renderFindResults(array $results) {
	$drequest = $this->getDiffusionRequest();

	$rows = array();
	foreach ($results as $result) {
	$href = $drequest->generateURI(array(
	'action' => 'browse',
	'path' => $result,
	));

	$readable = Filesystem::readablePath($result, $drequest->getPath());

	$rows[] = array(
	phutil_tag('a', array('href' => $href), $readable),
	);
	}

	$table = id(new AphrontTableView($rows))
	->setHeaders(array(pht('Path')))
	->setColumnClasses(array('wide'))
	->setNoDataString(
	pht(
	'The pattern you searched for did not match the names of any '.
	'files.'));

	return $table;
	}

	}
	diff --git a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php
	index c4d509fdb8..080a69c306 100644
	--- a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php
	+++ b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php
	@@ -1,1189 +1,1193 @@
	<?php

	/**
	* @task config Configuring the Hook Engine
	* @task hook Hook Execution
	* @task git Git Hooks
	* @task hg Mercurial Hooks
	* @task svn Subversion Hooks
	* @task internal Internals
	*/
	final class DiffusionCommitHookEngine extends Phobject {

	const ENV_USER = 'PHABRICATOR_USER';
	const ENV_REMOTE_ADDRESS = 'PHABRICATOR_REMOTE_ADDRESS';
	const ENV_REMOTE_PROTOCOL = 'PHABRICATOR_REMOTE_PROTOCOL';

	const EMPTY_HASH = '0000000000000000000000000000000000000000';

	private $viewer;
	private $repository;
	private $stdin;
	private $originalArgv;
	private $subversionTransaction;
	private $subversionRepository;
	private $remoteAddress;
	private $remoteProtocol;
	private $transactionKey;
	private $mercurialHook;
	private $mercurialCommits = array();
	private $gitCommits = array();

	private $heraldViewerProjects;
	private $rejectCode = PhabricatorRepositoryPushLog::REJECT_BROKEN;
	private $rejectDetails;
	private $emailPHIDs = array();


	/* -( Config )------------------------------------------------------------- */


	public function setRemoteProtocol($remote_protocol) {
	$this->remoteProtocol = $remote_protocol;
	return $this;
	}

	public function getRemoteProtocol() {
	return $this->remoteProtocol;
	}

	public function setRemoteAddress($remote_address) {
	$this->remoteAddress = $remote_address;
	return $this;
	}

	public function getRemoteAddress() {
	return $this->remoteAddress;
	}

	private function getRemoteAddressForLog() {
	// If whatever we have here isn't a valid IPv4 address, just store `null`.
	// Older versions of PHP return `-1` on failure instead of `false`.
	$remote_address = $this->getRemoteAddress();
	$remote_address = max(0, ip2long($remote_address));
	$remote_address = nonempty($remote_address, null);
	return $remote_address;
	}

	public function setSubversionTransactionInfo($transaction, $repository) {
	$this->subversionTransaction = $transaction;
	$this->subversionRepository = $repository;
	return $this;
	}

	public function setStdin($stdin) {
	$this->stdin = $stdin;
	return $this;
	}

	public function getStdin() {
	return $this->stdin;
	}

	public function setOriginalArgv(array $original_argv) {
	$this->originalArgv = $original_argv;
	return $this;
	}

	public function getOriginalArgv() {
	return $this->originalArgv;
	}

	public function setRepository(PhabricatorRepository $repository) {
	$this->repository = $repository;
	return $this;
	}

	public function getRepository() {
	return $this->repository;
	}

	public function setViewer(PhabricatorUser $viewer) {
	$this->viewer = $viewer;
	return $this;
	}

	public function getViewer() {
	return $this->viewer;
	}

	public function setMercurialHook($mercurial_hook) {
	$this->mercurialHook = $mercurial_hook;
	return $this;
	}

	public function getMercurialHook() {
	return $this->mercurialHook;
	}


	/* -( Hook Execution )----------------------------------------------------- */


	public function execute() {
	$ref_updates = $this->findRefUpdates();
	$all_updates = $ref_updates;

	$caught = null;
	try {

	try {
	$this->rejectDangerousChanges($ref_updates);
	} catch (DiffusionCommitHookRejectException $ex) {
	// If we're rejecting dangerous changes, flag everything that we've
	// seen as rejected so it's clear that none of it was accepted.
	$this->rejectCode = PhabricatorRepositoryPushLog::REJECT_DANGEROUS;
	throw $ex;
	}

	$this->applyHeraldRefRules($ref_updates, $all_updates);

	$content_updates = $this->findContentUpdates($ref_updates);
	$all_updates = array_merge($all_updates, $content_updates);

	$this->applyHeraldContentRules($content_updates, $all_updates);

	// Run custom scripts in `hook.d/` directories.
	$this->applyCustomHooks($all_updates);

	// If we make it this far, we're accepting these changes. Mark all the
	// logs as accepted.
	$this->rejectCode = PhabricatorRepositoryPushLog::REJECT_ACCEPT;
	} catch (Exception $ex) {
	// We'll throw this again in a minute, but we want to save all the logs
	// first.
	$caught = $ex;
	}

	// Save all the logs no matter what the outcome was.
	$event = $this->newPushEvent();

	$event->setRejectCode($this->rejectCode);
	$event->setRejectDetails($this->rejectDetails);

	$event->openTransaction();
	$event->save();
	foreach ($all_updates as $update) {
	$update->setPushEventPHID($event->getPHID());
	$update->save();
	}
	$event->saveTransaction();

	if ($caught) {
	throw $caught;
	}

	if ($this->emailPHIDs) {
	// If Herald rules triggered email to users, queue a worker to send the
	// mail. We do this out-of-process so that we block pushes as briefly
	// as possible.

	// (We do need to pull some commit info here because the commit objects
	// may not exist yet when this worker runs, which could be immediately.)

	PhabricatorWorker::scheduleTask(
	'PhabricatorRepositoryPushMailWorker',
	array(
	'eventPHID' => $event->getPHID(),
	'emailPHIDs' => array_values($this->emailPHIDs),
	'info' => $this->loadCommitInfoForWorker($all_updates),
	),
	array(
	'priority' => PhabricatorWorker::PRIORITY_ALERTS,
	));
	}

	return 0;
	}

	private function findRefUpdates() {
	$type = $this->getRepository()->getVersionControlSystem();
	switch ($type) {
	case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
	return $this->findGitRefUpdates();
	case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
	return $this->findMercurialRefUpdates();
	case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
	return $this->findSubversionRefUpdates();
	default:
	throw new Exception(pht('Unsupported repository type "%s"!', $type));
	}
	}

	private function rejectDangerousChanges(array $ref_updates) {
	assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog');

	$repository = $this->getRepository();
	if ($repository->shouldAllowDangerousChanges()) {
	return;
	}

	$flag_dangerous = PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS;

	foreach ($ref_updates as $ref_update) {
	if (!$ref_update->hasChangeFlags($flag_dangerous)) {
	// This is not a dangerous change.
	continue;
	}

	// We either have a branch deletion or a non fast-forward branch update.
	// Format a message and reject the push.

	$message = pht(
	"DANGEROUS CHANGE: %s\n".
	"Dangerous change protection is enabled for this repository.\n".
	"Edit the repository configuration before making dangerous changes.",
	$ref_update->getDangerousChangeDescription());

	throw new DiffusionCommitHookRejectException($message);
	}
	}

	private function findContentUpdates(array $ref_updates) {
	assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog');

	$type = $this->getRepository()->getVersionControlSystem();
	switch ($type) {
	case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
	return $this->findGitContentUpdates($ref_updates);
	case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
	return $this->findMercurialContentUpdates($ref_updates);
	case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
	return $this->findSubversionContentUpdates($ref_updates);
	default:
	throw new Exception(pht('Unsupported repository type "%s"!', $type));
	}
	}


	/* -( Herald )------------------------------------------------------------- */

	private function applyHeraldRefRules(
	array $ref_updates,
	array $all_updates) {
	$this->applyHeraldRules(
	$ref_updates,
	new HeraldPreCommitRefAdapter(),
	$all_updates);
	}

	private function applyHeraldContentRules(
	array $content_updates,
	array $all_updates) {
	$this->applyHeraldRules(
	$content_updates,
	new HeraldPreCommitContentAdapter(),
	$all_updates);
	}

	private function applyHeraldRules(
	array $updates,
	HeraldAdapter $adapter_template,
	array $all_updates) {

	if (!$updates) {
	return;
	}

	$adapter_template->setHookEngine($this);

	$engine = new HeraldEngine();
	$rules = null;
	$blocking_effect = null;
	$blocked_update = null;
	foreach ($updates as $update) {
	$adapter = id(clone $adapter_template)
	->setPushLog($update);

	if ($rules === null) {
	$rules = $engine->loadRulesForAdapter($adapter);
	}

	$effects = $engine->applyRules($rules, $adapter);
	$engine->applyEffects($effects, $adapter, $rules);
	$xscript = $engine->getTranscript();

	// Store any PHIDs we want to send email to for later.
	foreach ($adapter->getEmailPHIDs() as $email_phid) {
	$this->emailPHIDs[$email_phid] = $email_phid;
	}

	if ($blocking_effect === null) {
	foreach ($effects as $effect) {
	if ($effect->getAction() == HeraldAdapter::ACTION_BLOCK) {
	$blocking_effect = $effect;
	$blocked_update = $update;
	break;
	}
	}
	}
	}

	if ($blocking_effect) {
	$this->rejectCode = PhabricatorRepositoryPushLog::REJECT_HERALD;
	$this->rejectDetails = $blocking_effect->getRulePHID();

	$message = $blocking_effect->getTarget();
	if (!strlen($message)) {
	$message = pht('(None.)');
	}

	$rules = mpull($rules, null, 'getID');
	$rule = idx($rules, $effect->getRuleID());
	if ($rule && strlen($rule->getName())) {
	$rule_name = $rule->getName();
	} else {
	$rule_name = pht('Unnamed Herald Rule');
	}

	$blocked_ref_name = coalesce(
	$blocked_update->getRefName(),
	$blocked_update->getRefNewShort());
	$blocked_name = $blocked_update->getRefType().'/'.$blocked_ref_name;

	throw new DiffusionCommitHookRejectException(
	pht(
	"This push was rejected by Herald push rule %s.\n".
	"Change: %s\n".
	" Rule: %s\n".
	"Reason: %s",
	'H'.$blocking_effect->getRuleID(),
	$blocked_name,
	$rule_name,
	$message));
	}
	}

	public function loadViewerProjectPHIDsForHerald() {
	// This just caches the viewer's projects so we don't need to load them
	// over and over again when applying Herald rules.
	if ($this->heraldViewerProjects === null) {
	$this->heraldViewerProjects = id(new PhabricatorProjectQuery())
	->setViewer($this->getViewer())
	->withMemberPHIDs(array($this->getViewer()->getPHID()))
	->execute();
	}

	return mpull($this->heraldViewerProjects, 'getPHID');
	}


	/* -( Git )---------------------------------------------------------------- */


	private function findGitRefUpdates() {
	$ref_updates = array();

	// First, parse stdin, which lists all the ref changes. The input looks
	// like this:
	//
	// <old hash> <new hash> <ref>

	$stdin = $this->getStdin();
	$lines = phutil_split_lines($stdin, $retain_endings = false);
	foreach ($lines as $line) {
	$parts = explode(' ', $line, 3);
	if (count($parts) != 3) {
	throw new Exception(pht('Expected "old new ref", got "%s".', $line));
	}

	$ref_old = $parts[0];
	$ref_new = $parts[1];
	$ref_raw = $parts[2];

	if (preg_match('(^refs/heads/)', $ref_raw)) {
	$ref_type = PhabricatorRepositoryPushLog::REFTYPE_BRANCH;
	$ref_raw = substr($ref_raw, strlen('refs/heads/'));
	} else if (preg_match('(^refs/tags/)', $ref_raw)) {
	$ref_type = PhabricatorRepositoryPushLog::REFTYPE_TAG;
	$ref_raw = substr($ref_raw, strlen('refs/tags/'));
	} else {
	throw new Exception(
	pht(
	"Unable to identify the reftype of '%s'. Rejecting push.",
	$ref_raw));
	}

	$ref_update = $this->newPushLog()
	->setRefType($ref_type)
	->setRefName($ref_raw)
	->setRefOld($ref_old)
	->setRefNew($ref_new);

	$ref_updates[] = $ref_update;
	}

	$this->findGitMergeBases($ref_updates);
	$this->findGitChangeFlags($ref_updates);

	return $ref_updates;
	}


	private function findGitMergeBases(array $ref_updates) {
	assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog');

	$futures = array();
	foreach ($ref_updates as $key => $ref_update) {
	// If the old hash is "00000...", the ref is being created (either a new
	// branch, or a new tag). If the new hash is "00000...", the ref is being
	// deleted. If both are nonempty, the ref is being updated. For updates,
	// we'll figure out the `merge-base` of the old and new objects here. This
	// lets us reject non-FF changes cheaply; later, we'll figure out exactly
	// which commits are new.
	$ref_old = $ref_update->getRefOld();
	$ref_new = $ref_update->getRefNew();

	if (($ref_old === self::EMPTY_HASH) \|\|
	($ref_new === self::EMPTY_HASH)) {
	continue;
	}

	$futures[$key] = $this->getRepository()->getLocalCommandFuture(
	'merge-base %s %s',
	$ref_old,
	$ref_new);
	}

	- foreach (Futures($futures)->limit(8) as $key => $future) {
	+ $futures = id(new FutureIterator($futures))
	+ ->limit(8);
	+ foreach ($futures as $key => $future) {

	// If 'old' and 'new' have no common ancestors (for example, a force push
	// which completely rewrites a ref), `git merge-base` will exit with
	// an error and no output. It would be nice to find a positive test
	// for this instead, but I couldn't immediately come up with one. See
	// T4224. Assume this means there are no ancestors.

	list($err, $stdout) = $future->resolve();

	if ($err) {
	$merge_base = null;
	} else {
	$merge_base = rtrim($stdout, "\n");
	}

	$ref_update = $ref_updates[$key];
	$ref_update->setMergeBase($merge_base);
	}

	return $ref_updates;
	}


	private function findGitChangeFlags(array $ref_updates) {
	assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog');

	foreach ($ref_updates as $key => $ref_update) {
	$ref_old = $ref_update->getRefOld();
	$ref_new = $ref_update->getRefNew();
	$ref_type = $ref_update->getRefType();

	$ref_flags = 0;
	$dangerous = null;

	if (($ref_old === self::EMPTY_HASH) && ($ref_new === self::EMPTY_HASH)) {
	// This happens if you try to delete a tag or branch which does not
	// exist by pushing directly to the ref. Git will warn about it but
	// allow it. Just call it a delete, without flagging it as dangerous.
	$ref_flags \|= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE;
	} else if ($ref_old === self::EMPTY_HASH) {
	$ref_flags \|= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD;
	} else if ($ref_new === self::EMPTY_HASH) {
	$ref_flags \|= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE;
	if ($ref_type == PhabricatorRepositoryPushLog::REFTYPE_BRANCH) {
	$ref_flags \|= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS;
	$dangerous = pht(
	"The change you're attempting to push deletes the branch '%s'.",
	$ref_update->getRefName());
	}
	} else {
	$merge_base = $ref_update->getMergeBase();
	if ($merge_base == $ref_old) {
	// This is a fast-forward update to an existing branch.
	// These are safe.
	$ref_flags \|= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND;
	} else {
	$ref_flags \|= PhabricatorRepositoryPushLog::CHANGEFLAG_REWRITE;

	// For now, we don't consider deleting or moving tags to be a
	// "dangerous" update. It's way harder to get wrong and should be easy
	// to recover from once we have better logging. Only add the dangerous
	// flag if this ref is a branch.

	if ($ref_type == PhabricatorRepositoryPushLog::REFTYPE_BRANCH) {
	$ref_flags \|= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS;

	$dangerous = pht(
	"The change you're attempting to push updates the branch '%s' ".
	"from '%s' to '%s', but this is not a fast-forward. Pushes ".
	"which rewrite published branch history are dangerous.",
	$ref_update->getRefName(),
	$ref_update->getRefOldShort(),
	$ref_update->getRefNewShort());
	}
	}
	}

	$ref_update->setChangeFlags($ref_flags);
	if ($dangerous !== null) {
	$ref_update->attachDangerousChangeDescription($dangerous);
	}
	}

	return $ref_updates;
	}


	private function findGitContentUpdates(array $ref_updates) {
	$flag_delete = PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE;

	$futures = array();
	foreach ($ref_updates as $key => $ref_update) {
	if ($ref_update->hasChangeFlags($flag_delete)) {
	// Deleting a branch or tag can never create any new commits.
	continue;
	}

	// NOTE: This piece of magic finds all new commits, by walking backward
	// from the new value to the value of any existing ref in the
	// repository. Particularly, this will cover the cases of a new branch, a
	// completely moved tag, etc.
	$futures[$key] = $this->getRepository()->getLocalCommandFuture(
	'log --format=%s %s --not --all',
	'%H',
	$ref_update->getRefNew());
	}

	$content_updates = array();
	- foreach (Futures($futures)->limit(8) as $key => $future) {
	+ $futures = id(new FutureIterator($futures))
	+ ->limit(8);
	+ foreach ($futures as $key => $future) {
	list($stdout) = $future->resolvex();

	if (!strlen(trim($stdout))) {
	// This change doesn't have any new commits. One common case of this
	// is creating a new tag which points at an existing commit.
	continue;
	}

	$commits = phutil_split_lines($stdout, $retain_newlines = false);

	// If we're looking at a branch, mark all of the new commits as on that
	// branch. It's only possible for these commits to be on updated branches,
	// since any other branch heads are necessarily behind them.
	$branch_name = null;
	$ref_update = $ref_updates[$key];
	$type_branch = PhabricatorRepositoryPushLog::REFTYPE_BRANCH;
	if ($ref_update->getRefType() == $type_branch) {
	$branch_name = $ref_update->getRefName();
	}

	foreach ($commits as $commit) {
	if ($branch_name) {
	$this->gitCommits[$commit][] = $branch_name;
	}
	$content_updates[$commit] = $this->newPushLog()
	->setRefType(PhabricatorRepositoryPushLog::REFTYPE_COMMIT)
	->setRefNew($commit)
	->setChangeFlags(PhabricatorRepositoryPushLog::CHANGEFLAG_ADD);
	}
	}

	return $content_updates;
	}

	/* -( Custom )------------------------------------------------------------- */

	private function applyCustomHooks(array $updates) {
	$args = $this->getOriginalArgv();
	$stdin = $this->getStdin();
	$console = PhutilConsole::getConsole();

	$env = array(
	'PHABRICATOR_REPOSITORY' => $this->getRepository()->getCallsign(),
	self::ENV_USER => $this->getViewer()->getUsername(),
	self::ENV_REMOTE_PROTOCOL => $this->getRemoteProtocol(),
	self::ENV_REMOTE_ADDRESS => $this->getRemoteAddress(),
	);

	$directories = $this->getRepository()->getHookDirectories();
	foreach ($directories as $directory) {
	$hooks = $this->getExecutablesInDirectory($directory);
	sort($hooks);
	foreach ($hooks as $hook) {
	// NOTE: We're explicitly running the hooks in sequential order to
	// make this more predictable.
	$future = id(new ExecFuture('%s %Ls', $hook, $args))
	->setEnv($env, $wipe_process_env = false)
	->write($stdin);

	list($err, $stdout, $stderr) = $future->resolve();
	if (!$err) {
	// This hook ran OK, but echo its output in case there was something
	// informative.
	$console->writeOut('%s', $stdout);
	$console->writeErr('%s', $stderr);
	continue;
	}

	$this->rejectCode = PhabricatorRepositoryPushLog::REJECT_EXTERNAL;
	$this->rejectDetails = basename($hook);

	throw new DiffusionCommitHookRejectException(
	pht(
	"This push was rejected by custom hook script '%s':\n\n%s%s",
	basename($hook),
	$stdout,
	$stderr));
	}
	}
	}

	private function getExecutablesInDirectory($directory) {
	$executables = array();

	if (!Filesystem::pathExists($directory)) {
	return $executables;
	}

	foreach (Filesystem::listDirectory($directory) as $path) {
	$full_path = $directory.DIRECTORY_SEPARATOR.$path;
	if (!is_executable($full_path)) {
	// Don't include non-executable files.
	continue;
	}

	if (basename($full_path) == 'README') {
	// Don't include README, even if it is marked as executable. It almost
	// certainly got caught in the crossfire of a sweeping `chmod`, since
	// users do this with some frequency.
	continue;
	}

	$executables[] = $full_path;
	}

	return $executables;
	}


	/* -( Mercurial )---------------------------------------------------------- */


	private function findMercurialRefUpdates() {
	$hook = $this->getMercurialHook();
	switch ($hook) {
	case 'pretxnchangegroup':
	return $this->findMercurialChangegroupRefUpdates();
	case 'prepushkey':
	return $this->findMercurialPushKeyRefUpdates();
	default:
	throw new Exception(pht('Unrecognized hook "%s"!', $hook));
	}
	}

	private function findMercurialChangegroupRefUpdates() {
	$hg_node = getenv('HG_NODE');
	if (!$hg_node) {
	throw new Exception(pht('Expected HG_NODE in environment!'));
	}

	// NOTE: We need to make sure this is passed to subprocesses, or they won't
	// be able to see new commits. Mercurial uses this as a marker to determine
	// whether the pending changes are visible or not.
	$_ENV['HG_PENDING'] = getenv('HG_PENDING');
	$repository = $this->getRepository();

	$futures = array();

	foreach (array('old', 'new') as $key) {
	$futures[$key] = $repository->getLocalCommandFuture(
	'heads --template %s',
	'{node}\1{branch}\2');
	}
	// Wipe HG_PENDING out of the old environment so we see the pre-commit
	// state of the repository.
	$futures['old']->updateEnv('HG_PENDING', null);

	$futures['commits'] = $repository->getLocalCommandFuture(
	'log --rev %s --template %s',
	hgsprintf('%s:%s', $hg_node, 'tip'),
	'{node}\1{branch}\2');

	// Resolve all of the futures now. We don't need the 'commits' future yet,
	// but it simplifies the logic to just get it out of the way.
	- foreach (Futures($futures) as $future) {
	+ foreach (new FutureIterator($futures) as $future) {
	$future->resolve();
	}

	list($commit_raw) = $futures['commits']->resolvex();
	$commit_map = $this->parseMercurialCommits($commit_raw);
	$this->mercurialCommits = $commit_map;

	// NOTE: `hg heads` exits with an error code and no output if the repository
	// has no heads. Most commonly this happens on a new repository. We know
	// we can run `hg` successfully since the `hg log` above didn't error, so
	// just ignore the error code.

	list($err, $old_raw) = $futures['old']->resolve();
	$old_refs = $this->parseMercurialHeads($old_raw);

	list($err, $new_raw) = $futures['new']->resolve();
	$new_refs = $this->parseMercurialHeads($new_raw);

	$all_refs = array_keys($old_refs + $new_refs);

	$ref_updates = array();
	foreach ($all_refs as $ref) {
	$old_heads = idx($old_refs, $ref, array());
	$new_heads = idx($new_refs, $ref, array());

	sort($old_heads);
	sort($new_heads);

	if (!$old_heads && !$new_heads) {
	// This should never be possible, as it makes no sense. Explode.
	throw new Exception(
	pht(
	'Mercurial repository has no new or old heads for branch "%s" '.
	'after push. This makes no sense; rejecting change.',
	$ref));
	}

	if ($old_heads === $new_heads) {
	// No changes to this branch, so skip it.
	continue;
	}

	$stray_heads = array();

	if ($old_heads && !$new_heads) {
	// This is a branch deletion with "--close-branch".
	$head_map = array();
	foreach ($old_heads as $old_head) {
	$head_map[$old_head] = array(self::EMPTY_HASH);
	}
	} else if (count($old_heads) > 1) {
	// HORRIBLE: In Mercurial, branches can have multiple heads. If the
	// old branch had multiple heads, we need to figure out which new
	// heads descend from which old heads, so we can tell whether you're
	// actively creating new heads (dangerous) or just working in a
	// repository that's already full of garbage (strongly discouraged but
	// not as inherently dangerous). These cases should be very uncommon.

	// NOTE: We're only looking for heads on the same branch. The old
	// tip of the branch may be the branchpoint for other branches, but that
	// is OK.

	$dfutures = array();
	foreach ($old_heads as $old_head) {
	$dfutures[$old_head] = $repository->getLocalCommandFuture(
	'log --branch %s --rev %s --template %s',
	$ref,
	hgsprintf('(descendants(%s) and head())', $old_head),
	'{node}\1');
	}

	$head_map = array();
	- foreach (Futures($dfutures) as $future_head => $dfuture) {
	+ foreach (new FutureIterator($dfutures) as $future_head => $dfuture) {
	list($stdout) = $dfuture->resolvex();
	$descendant_heads = array_filter(explode("\1", $stdout));
	if ($descendant_heads) {
	// This old head has at least one descendant in the push.
	$head_map[$future_head] = $descendant_heads;
	} else {
	// This old head has no descendants, so it is being deleted.
	$head_map[$future_head] = array(self::EMPTY_HASH);
	}
	}

	// Now, find all the new stray heads this push creates, if any. These
	// are new heads which do not descend from the old heads.
	$seen = array_fuse(array_mergev($head_map));
	foreach ($new_heads as $new_head) {
	if ($new_head === self::EMPTY_HASH) {
	// If a branch head is being deleted, don't insert it as an add.
	continue;
	}
	if (empty($seen[$new_head])) {
	$head_map[self::EMPTY_HASH][] = $new_head;
	}
	}
	} else if ($old_heads) {
	$head_map[head($old_heads)] = $new_heads;
	} else {
	$head_map[self::EMPTY_HASH] = $new_heads;
	}

	foreach ($head_map as $old_head => $child_heads) {
	foreach ($child_heads as $new_head) {
	if ($new_head === $old_head) {
	continue;
	}

	$ref_flags = 0;
	$dangerous = null;
	if ($old_head == self::EMPTY_HASH) {
	$ref_flags \|= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD;
	} else {
	$ref_flags \|= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND;
	}


	$deletes_existing_head = ($new_head == self::EMPTY_HASH);
	$splits_existing_head = (count($child_heads) > 1);
	$creates_duplicate_head = ($old_head == self::EMPTY_HASH) &&
	(count($head_map) > 1);

	if ($splits_existing_head \|\| $creates_duplicate_head) {
	$readable_child_heads = array();
	foreach ($child_heads as $child_head) {
	$readable_child_heads[] = substr($child_head, 0, 12);
	}

	$ref_flags \|= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS;

	if ($splits_existing_head) {
	// We're splitting an existing head into two or more heads.
	// This is dangerous, and a super bad idea. Note that we're only
	// raising this if you're actively splitting a branch head. If a
	// head split in the past, we don't consider appends to it
	// to be dangerous.
	$dangerous = pht(
	"The change you're attempting to push splits the head of ".
	"branch '%s' into multiple heads: %s. This is inadvisable ".
	"and dangerous.",
	$ref,
	implode(', ', $readable_child_heads));
	} else {
	// We're adding a second (or more) head to a branch. The new
	// head is not a descendant of any old head.
	$dangerous = pht(
	"The change you're attempting to push creates new, divergent ".
	"heads for the branch '%s': %s. This is inadvisable and ".
	"dangerous.",
	$ref,
	implode(', ', $readable_child_heads));
	}
	}

	if ($deletes_existing_head) {
	// TODO: Somewhere in here we should be setting CHANGEFLAG_REWRITE
	// if we are also creating at least one other head to replace
	// this one.

	// NOTE: In Git, this is a dangerous change, but it is not dangerous
	// in Mercurial. Mercurial branches are version controlled, and
	// Mercurial does not prompt you for any special flags when pushing
	// a `--close-branch` commit by default.

	$ref_flags \|= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE;
	}

	$ref_update = $this->newPushLog()
	->setRefType(PhabricatorRepositoryPushLog::REFTYPE_BRANCH)
	->setRefName($ref)
	->setRefOld($old_head)
	->setRefNew($new_head)
	->setChangeFlags($ref_flags);

	if ($dangerous !== null) {
	$ref_update->attachDangerousChangeDescription($dangerous);
	}

	$ref_updates[] = $ref_update;
	}
	}
	}

	return $ref_updates;
	}

	private function findMercurialPushKeyRefUpdates() {
	$key_namespace = getenv('HG_NAMESPACE');

	if ($key_namespace === 'phases') {
	// Mercurial changes commit phases as part of normal push operations. We
	// just ignore these, as they don't seem to represent anything
	// interesting.
	return array();
	}

	$key_name = getenv('HG_KEY');

	$key_old = getenv('HG_OLD');
	if (!strlen($key_old)) {
	$key_old = null;
	}

	$key_new = getenv('HG_NEW');
	if (!strlen($key_new)) {
	$key_new = null;
	}

	if ($key_namespace !== 'bookmarks') {
	throw new Exception(
	pht(
	"Unknown Mercurial key namespace '%s', with key '%s' (%s -> %s). ".
	"Rejecting push.",
	$key_namespace,
	$key_name,
	coalesce($key_old, pht('null')),
	coalesce($key_new, pht('null'))));
	}

	if ($key_old === $key_new) {
	// We get a callback when the bookmark doesn't change. Just ignore this,
	// as it's a no-op.
	return array();
	}

	$ref_flags = 0;
	$merge_base = null;
	if ($key_old === null) {
	$ref_flags \|= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD;
	} else if ($key_new === null) {
	$ref_flags \|= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE;
	} else {
	list($merge_base_raw) = $this->getRepository()->execxLocalCommand(
	'log --template %s --rev %s',
	'{node}',
	hgsprintf('ancestor(%s, %s)', $key_old, $key_new));

	if (strlen(trim($merge_base_raw))) {
	$merge_base = trim($merge_base_raw);
	}

	if ($merge_base && ($merge_base === $key_old)) {
	$ref_flags \|= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND;
	} else {
	$ref_flags \|= PhabricatorRepositoryPushLog::CHANGEFLAG_REWRITE;
	}
	}

	$ref_update = $this->newPushLog()
	->setRefType(PhabricatorRepositoryPushLog::REFTYPE_BOOKMARK)
	->setRefName($key_name)
	->setRefOld(coalesce($key_old, self::EMPTY_HASH))
	->setRefNew(coalesce($key_new, self::EMPTY_HASH))
	->setChangeFlags($ref_flags);

	return array($ref_update);
	}

	private function findMercurialContentUpdates(array $ref_updates) {
	$content_updates = array();

	foreach ($this->mercurialCommits as $commit => $branches) {
	$content_updates[$commit] = $this->newPushLog()
	->setRefType(PhabricatorRepositoryPushLog::REFTYPE_COMMIT)
	->setRefNew($commit)
	->setChangeFlags(PhabricatorRepositoryPushLog::CHANGEFLAG_ADD);
	}

	return $content_updates;
	}

	private function parseMercurialCommits($raw) {
	$commits_lines = explode("\2", $raw);
	$commits_lines = array_filter($commits_lines);
	$commit_map = array();
	foreach ($commits_lines as $commit_line) {
	list($node, $branch) = explode("\1", $commit_line);
	$commit_map[$node] = array($branch);
	}

	return $commit_map;
	}

	private function parseMercurialHeads($raw) {
	$heads_map = $this->parseMercurialCommits($raw);

	$heads = array();
	foreach ($heads_map as $commit => $branches) {
	foreach ($branches as $branch) {
	$heads[$branch][] = $commit;
	}
	}

	return $heads;
	}


	/* -( Subversion )--------------------------------------------------------- */


	private function findSubversionRefUpdates() {
	// Subversion doesn't have any kind of mutable ref metadata.
	return array();
	}

	private function findSubversionContentUpdates(array $ref_updates) {
	list($youngest) = execx(
	'svnlook youngest %s',
	$this->subversionRepository);
	$ref_new = (int)$youngest + 1;

	$ref_flags = 0;
	$ref_flags \|= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD;
	$ref_flags \|= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND;

	$ref_content = $this->newPushLog()
	->setRefType(PhabricatorRepositoryPushLog::REFTYPE_COMMIT)
	->setRefNew($ref_new)
	->setChangeFlags($ref_flags);

	return array($ref_content);
	}


	/* -( Internals )---------------------------------------------------------- */


	private function newPushLog() {
	// NOTE: We generate PHIDs up front so the Herald transcripts can pick them
	// up.
	$phid = id(new PhabricatorRepositoryPushLog())->generatePHID();

	return PhabricatorRepositoryPushLog::initializeNewLog($this->getViewer())
	->setPHID($phid)
	->setRepositoryPHID($this->getRepository()->getPHID())
	->attachRepository($this->getRepository())
	->setEpoch(time());
	}

	private function newPushEvent() {
	$viewer = $this->getViewer();
	return PhabricatorRepositoryPushEvent::initializeNewEvent($viewer)
	->setRepositoryPHID($this->getRepository()->getPHID())
	->setRemoteAddress($this->getRemoteAddressForLog())
	->setRemoteProtocol($this->getRemoteProtocol())
	->setEpoch(time());
	}

	public function loadChangesetsForCommit($identifier) {
	$byte_limit = HeraldCommitAdapter::getEnormousByteLimit();
	$time_limit = HeraldCommitAdapter::getEnormousTimeLimit();

	$vcs = $this->getRepository()->getVersionControlSystem();
	switch ($vcs) {
	case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
	case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
	// For git and hg, we can use normal commands.
	$drequest = DiffusionRequest::newFromDictionary(
	array(
	'repository' => $this->getRepository(),
	'user' => $this->getViewer(),
	'commit' => $identifier,
	));

	$raw_diff = DiffusionRawDiffQuery::newFromDiffusionRequest($drequest)
	->setTimeout($time_limit)
	->setByteLimit($byte_limit)
	->setLinesOfContext(0)
	->loadRawDiff();
	break;
	case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
	// TODO: This diff has 3 lines of context, which produces slightly
	// incorrect "added file content" and "removed file content" results.
	// This may also choke on binaries, but "svnlook diff" does not support
	// the "--diff-cmd" flag.

	// For subversion, we need to use `svnlook`.
	$future = new ExecFuture(
	'svnlook diff -t %s %s',
	$this->subversionTransaction,
	$this->subversionRepository);

	$future->setTimeout($time_limit);
	$future->setStdoutSizeLimit($byte_limit);
	$future->setStderrSizeLimit($byte_limit);

	list($raw_diff) = $future->resolvex();
	break;
	default:
	throw new Exception(pht("Unknown VCS '%s!'", $vcs));
	}

	if (strlen($raw_diff) >= $byte_limit) {
	throw new Exception(
	pht(
	'The raw text of this change is enormous (larger than %d '.
	'bytes). Herald can not process it.',
	$byte_limit));
	}

	if (!strlen($raw_diff)) {
	// If the commit is actually empty, just return no changesets.
	return array();
	}

	$parser = new ArcanistDiffParser();
	$changes = $parser->parseDiff($raw_diff);
	$diff = DifferentialDiff::newEphemeralFromRawChanges(
	$changes);
	return $diff->getChangesets();
	}

	public function loadCommitRefForCommit($identifier) {
	$repository = $this->getRepository();
	$vcs = $repository->getVersionControlSystem();
	switch ($vcs) {
	case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
	case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
	return id(new DiffusionLowLevelCommitQuery())
	->setRepository($repository)
	->withIdentifier($identifier)
	->execute();
	case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
	// For subversion, we need to use `svnlook`.
	list($message) = execx(
	'svnlook log -t %s %s',
	$this->subversionTransaction,
	$this->subversionRepository);

	return id(new DiffusionCommitRef())
	->setMessage($message);
	break;
	default:
	throw new Exception(pht("Unknown VCS '%s!'", $vcs));
	}
	}

	public function loadBranches($identifier) {
	$repository = $this->getRepository();
	$vcs = $repository->getVersionControlSystem();
	switch ($vcs) {
	case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
	return idx($this->gitCommits, $identifier, array());
	case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
	// NOTE: This will be "the branch the commit was made to", not
	// "a list of all branch heads which descend from the commit".
	// This is consistent with Mercurial, but possibly confusing.
	return idx($this->mercurialCommits, $identifier, array());
	case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
	// Subversion doesn't have branches.
	return array();
	}
	}

	private function loadCommitInfoForWorker(array $all_updates) {
	$type_commit = PhabricatorRepositoryPushLog::REFTYPE_COMMIT;

	$map = array();
	foreach ($all_updates as $update) {
	if ($update->getRefType() != $type_commit) {
	continue;
	}
	$map[$update->getRefNew()] = array();
	}

	foreach ($map as $identifier => $info) {
	$ref = $this->loadCommitRefForCommit($identifier);
	$map[$identifier] += array(
	'summary' => $ref->getSummary(),
	'branches' => $this->loadBranches($identifier),
	);
	}

	return $map;
	}

	}
	diff --git a/src/applications/diffusion/query/lowlevel/DiffusionLowLevelResolveRefsQuery.php b/src/applications/diffusion/query/lowlevel/DiffusionLowLevelResolveRefsQuery.php
	index ca47c04ca1..72f2d18533 100644
	--- a/src/applications/diffusion/query/lowlevel/DiffusionLowLevelResolveRefsQuery.php
	+++ b/src/applications/diffusion/query/lowlevel/DiffusionLowLevelResolveRefsQuery.php
	@@ -1,219 +1,219 @@
	<?php

	/**
	* Resolves references (like short commit names, branch names, tag names, etc.)
	* into canonical, stable commit identifiers. This query works for all
	* repository types.
	*/
	final class DiffusionLowLevelResolveRefsQuery
	extends DiffusionLowLevelQuery {

	private $refs;

	public function withRefs(array $refs) {
	$this->refs = $refs;
	return $this;
	}

	public function executeQuery() {
	if (!$this->refs) {
	return array();
	}

	switch ($this->getRepository()->getVersionControlSystem()) {
	case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
	$result = $this->resolveGitRefs();
	break;
	case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
	$result = $this->resolveMercurialRefs();
	break;
	case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
	$result = $this->resolveSubversionRefs();
	break;
	default:
	throw new Exception('Unsupported repository type!');
	}

	return $result;
	}

	private function resolveGitRefs() {
	$repository = $this->getRepository();

	$future = $repository->getLocalCommandFuture('cat-file --batch-check');
	$future->write(implode("\n", $this->refs));
	list($stdout) = $future->resolvex();

	$lines = explode("\n", rtrim($stdout, "\n"));
	if (count($lines) !== count($this->refs)) {
	throw new Exception('Unexpected line count from `git cat-file`!');
	}

	$hits = array();
	$tags = array();

	$lines = array_combine($this->refs, $lines);
	foreach ($lines as $ref => $line) {
	$parts = explode(' ', $line);
	if (count($parts) < 2) {
	throw new Exception("Failed to parse `git cat-file` output: {$line}");
	}
	list($identifier, $type) = $parts;

	if ($type == 'missing') {
	// This is either an ambiguous reference which resolves to several
	// objects, or an invalid reference. For now, always treat it as
	// invalid. It would be nice to resolve all possibilities for
	// ambiguous references at some point, although the strategy for doing
	// so isn't clear to me.
	continue;
	}

	switch ($type) {
	case 'commit':
	break;
	case 'tag':
	$tags[] = $identifier;
	break;
	default:
	throw new Exception(
	"Unexpected object type from `git cat-file`: {$line}");
	}

	$hits[] = array(
	'ref' => $ref,
	'type' => $type,
	'identifier' => $identifier,
	);
	}

	$tag_map = array();
	if ($tags) {
	// If some of the refs were tags, just load every tag in order to figure
	// out which commits they map to. This might be somewhat inefficient in
	// repositories with a huge number of tags.
	$tag_refs = id(new DiffusionLowLevelGitRefQuery())
	->setRepository($repository)
	->withIsTag(true)
	->executeQuery();
	foreach ($tag_refs as $tag_ref) {
	$tag_map[$tag_ref->getShortName()] = $tag_ref->getCommitIdentifier();
	}
	}

	$results = array();
	foreach ($hits as $hit) {
	$type = $hit['type'];
	$ref = $hit['ref'];

	$alternate = null;
	if ($type == 'tag') {
	$alternate = $identifier;
	$identifier = idx($tag_map, $ref);
	if (!$identifier) {
	throw new Exception("Failed to look up tag '{$ref}'!");
	}
	}

	$result = array(
	'type' => $type,
	'identifier' => $identifier,
	);

	if ($alternate !== null) {
	$result['alternate'] = $alternate;
	}

	$results[$ref][] = $result;
	}

	return $results;
	}

	private function resolveMercurialRefs() {
	$repository = $this->getRepository();

	$futures = array();
	foreach ($this->refs as $ref) {
	$futures[$ref] = $repository->getLocalCommandFuture(
	'log --template=%s --rev %s',
	'{node}',
	hgsprintf('%s', $ref));
	}

	$results = array();
	- foreach (Futures($futures) as $ref => $future) {
	+ foreach (new FutureIterator($futures) as $ref => $future) {
	try {
	list($stdout) = $future->resolvex();
	} catch (CommandException $ex) {
	if (preg_match('/ambiguous identifier/', $ex->getStdErr())) {
	// This indicates that the ref ambiguously matched several things.
	// Eventually, it would be nice to return all of them, but it is
	// unclear how to best do that. For now, treat it as a miss instead.
	continue;
	}
	throw $ex;
	}

	// It doesn't look like we can figure out the type (commit/branch/rev)
	// from this output very easily. For now, just call everything a commit.
	$type = 'commit';

	$results[$ref][] = array(
	'type' => $type,
	'identifier' => trim($stdout),
	);
	}

	return $results;
	}

	private function resolveSubversionRefs() {
	$repository = $this->getRepository();

	$max_commit = id(new PhabricatorRepositoryCommit())
	->loadOneWhere(
	'repositoryID = %d ORDER BY epoch DESC, id DESC LIMIT 1',
	$repository->getID());
	if (!$max_commit) {
	// This repository is empty or hasn't parsed yet, so none of the refs are
	// going to resolve.
	return array();
	}

	$max_commit_id = (int)$max_commit->getCommitIdentifier();

	$results = array();
	foreach ($this->refs as $ref) {
	if ($ref == 'HEAD') {
	// Resolve "HEAD" to mean "the most recent commit".
	$results[$ref][] = array(
	'type' => 'commit',
	'identifier' => $max_commit_id,
	);
	continue;
	}

	if (!preg_match('/^\d+$/', $ref)) {
	// This ref is non-numeric, so it doesn't resolve to anything.
	continue;
	}

	// Resolve other commits if we can deduce their existence.

	// TODO: When we import only part of a repository, we won't necessarily
	// have all of the smaller commits. Should we fail to resolve them here
	// for repositories with a subpath? It might let us simplify other things
	// elsewhere.
	if ((int)$ref <= $max_commit_id) {
	$results[$ref][] = array(
	'type' => 'commit',
	'identifier' => (int)$ref,
	);
	}
	}

	return $results;
	}

	}
	diff --git a/src/applications/diviner/workflow/DivinerGenerateWorkflow.php b/src/applications/diviner/workflow/DivinerGenerateWorkflow.php
	index 7a75f744ab..bc799af303 100644
	--- a/src/applications/diviner/workflow/DivinerGenerateWorkflow.php
	+++ b/src/applications/diviner/workflow/DivinerGenerateWorkflow.php
	@@ -1,529 +1,531 @@
	<?php

	final class DivinerGenerateWorkflow extends DivinerWorkflow {

	private $atomCache;

	public function didConstruct() {
	$this
	->setName('generate')
	->setSynopsis(pht('Generate documentation.'))
	->setArguments(
	array(
	array(
	'name' => 'clean',
	'help' => 'Clear the caches before generating documentation.',
	),
	array(
	'name' => 'book',
	'param' => 'path',
	'help' => 'Path to a Diviner book configuration.',
	),
	));
	}

	protected function getAtomCache() {
	if (!$this->atomCache) {
	$book_root = $this->getConfig('root');
	$book_name = $this->getConfig('name');
	$cache_directory = $book_root.'/.divinercache/'.$book_name;
	$this->atomCache = new DivinerAtomCache($cache_directory);
	}
	return $this->atomCache;
	}

	protected function log($message) {
	$console = PhutilConsole::getConsole();
	$console->writeErr($message."\n");
	}

	public function execute(PhutilArgumentParser $args) {
	$book = $args->getArg('book');
	if ($book) {
	$books = array($book);
	} else {
	$cwd = getcwd();
	$this->log(pht('FINDING DOCUMENTATION BOOKS'));
	$books = id(new FileFinder($cwd))
	->withType('f')
	->withSuffix('book')
	->find();

	if (!$books) {
	throw new PhutilArgumentUsageException(
	pht(
	"There are no Diviner '.book' files anywhere beneath the ".
	"current directory. Use '--book <book>' to specify a ".
	"documentation book to generate."));
	} else {
	$this->log(pht('Found %s book(s).', new PhutilNumber(count($books))));
	}
	}

	foreach ($books as $book) {
	$short_name = basename($book);

	$this->log(pht('Generating book "%s"...', $short_name));
	$this->generateBook($book, $args);
	$this->log(pht('Completed generation of "%s".', $short_name)."\n");
	}
	}

	private function generateBook($book, PhutilArgumentParser $args) {
	$this->atomCache = null;

	$this->readBookConfiguration($book);

	if ($args->getArg('clean')) {
	$this->log(pht('CLEARING CACHES'));
	$this->getAtomCache()->delete();
	$this->log(pht('Done.')."\n");
	}

	// The major challenge of documentation generation is one of dependency
	// management. When regenerating documentation, we want to do the smallest
	// amount of work we can, so that regenerating documentation after minor
	// changes is quick.
	//
	// ATOM CACHE
	//
	// In the first stage, we find all the direct changes to source code since
	// the last run. This stage relies on two data structures:
	//
	// - File Hash Map: map<file_hash, node_hash>
	// - Atom Map: map<node_hash, true>
	//
	// First, we hash all the source files in the project to detect any which
	// have changed since the previous run (i.e., their hash is not present in
	// the File Hash Map). If a file's content hash appears in the map, it has
	// not changed, so we don't need to reparse it.
	//
	// We break the contents of each file into "atoms", which represent a unit
	// of source code (like a function, method, class or file). Each atom has a
	// "node hash" based on the content of the atom: if a function definition
	// changes, the node hash of the atom changes too. The primary output of
	// the atom cache is a list of node hashes which exist in the project. This
	// is the Atom Map. The node hash depends only on the definition of the atom
	// and the atomizer implementation. It ends with an "N", for "node".
	//
	// (We need the Atom Map in addition to the File Hash Map because each file
	// may have several atoms in it (e.g., multiple functions, or a class and
	// its methods). The File Hash Map contains an exhaustive list of all atoms
	// with type "file", but not child atoms of those top-level atoms.)
	//
	// GRAPH CACHE
	//
	// We now know which atoms exist, and can compare the Atom Map to some
	// existing cache to figure out what has changed. However, this isn't
	// sufficient to figure out which documentation actually needs to be
	// regnerated, because atoms depend on other atoms. For example, if "B
	// extends A" and the definition for A changes, we need to regenerate the
	// documentation in B. Similarly, if X links to Y and Y changes, we should
	// regenerate X. (In both these cases, the documentation for the connected
	// atom may not acutally change, but in some cases it will, and the extra
	// work we need to do is generally very small compared to the size of the
	// project.)
	//
	// To figure out which other nodes have changed, we compute a "graph hash"
	// for each node. This hash combines the "node hash" with the node hashes
	// of connected nodes. Our primary output is a list of graph hashes, which
	// a documentation generator can use to easily determine what work needs
	// to be done by comparing the list with a list of cached graph hashes,
	// then generating documentation for new hashes and deleting documentation
	// for missing hashes. The graph hash ends with a "G", for "graph".
	//
	// In this stage, we rely on three data structures:
	//
	// - Symbol Map: map<node_hash, symbol_hash>
	// - Edge Map: map<node_hash, list<symbol_hash>>
	// - Graph Map: map<node_hash, graph_hash>
	//
	// Calculating the graph hash requires several steps, because we need to
	// figure out which nodes an atom is attached to. The atom contains symbolic
	// references to other nodes by name (e.g., "extends SomeClass") in the form
	// of DivinerAtomRefs. We can also build a symbolic reference for any atom
	// from the atom itself. Each DivinerAtomRef generates a symbol hash,
	// which ends with an "S", for "symbol".
	//
	// First, we update the symbol map. We remove (and mark dirty) any symbols
	// associated with node hashes which no longer exist (e.g., old/dead nodes).
	// Second, we add (and mark dirty) any symbols associated with new nodes.
	// We also add edges defined by new nodes to the graph.
	//
	// We initialize a list of dirty nodes to the list of new nodes, then
	// find all nodes connected to dirty symbols and add them to the dirty
	// node list. This list now contains every node with a new or changed
	// graph hash.
	//
	// We walk the dirty list and compute the new graph hashes, adding them
	// to the graph hash map. This Graph Map can then be passed to an actual
	// documentation generator, which can compare the graph hashes to a list
	// of already-generated graph hashes and easily assess which documents need
	// to be regenerated and which can be deleted.

	$this->buildAtomCache();
	$this->buildGraphCache();

	$this->publishDocumentation($args->getArg('clean'));
	}

	/* -( Atom Cache )--------------------------------------------------------- */

	private function buildAtomCache() {
	$this->log(pht('BUILDING ATOM CACHE'));

	$file_hashes = $this->findFilesInProject();

	$this->log(pht('Found %d file(s) in project.', count($file_hashes)));

	$this->deleteDeadAtoms($file_hashes);

	$atomize = $this->getFilesToAtomize($file_hashes);

	$this->log(pht('Found %d unatomized, uncached file(s).', count($atomize)));

	$file_atomizers = $this->getAtomizersForFiles($atomize);

	$this->log(pht('Found %d file(s) to atomize.', count($file_atomizers)));

	$futures = $this->buildAtomizerFutures($file_atomizers);

	$this->log(pht('Atomizing %d file(s).', count($file_atomizers)));

	if ($futures) {
	$this->resolveAtomizerFutures($futures, $file_hashes);
	$this->log(pht('Atomization complete.'));
	} else {
	$this->log(pht('Atom cache is up to date, no files to atomize.'));
	}

	$this->log(pht('Writing atom cache.'));

	$this->getAtomCache()->saveAtoms();

	$this->log(pht('Done.')."\n");
	}

	private function getAtomizersForFiles(array $files) {
	$rules = $this->getRules();
	$exclude = $this->getExclude();

	$atomizers = array();

	foreach ($files as $file) {
	foreach ($exclude as $pattern) {
	if (preg_match($pattern, $file)) {
	continue 2;
	}
	}

	foreach ($rules as $rule => $atomizer) {
	$ok = preg_match($rule, $file);
	if ($ok === false) {
	throw new Exception(
	"Rule '{$rule}' is not a valid regular expression.");
	}
	if ($ok) {
	$atomizers[$file] = $atomizer;
	continue;
	}
	}
	}

	return $atomizers;
	}

	private function getRules() {
	$rules = $this->getConfig('rules', array(
	'/\\.diviner$/' => 'DivinerArticleAtomizer',
	'/\\.php$/' => 'DivinerPHPAtomizer',
	));

	return $rules;
	}

	private function getExclude() {
	$exclude = (array)$this->getConfig('exclude', array());
	return $exclude;
	}


	private function findFilesInProject() {
	$raw_hashes = id(new FileFinder($this->getConfig('root')))
	->excludePath('/.')
	->withType('f')
	->setGenerateChecksums(true)
	->find();

	$version = $this->getDivinerAtomWorldVersion();

	$file_hashes = array();
	foreach ($raw_hashes as $file => $md5_hash) {
	$rel_file = Filesystem::readablePath($file, $this->getConfig('root'));
	// We want the hash to change if the file moves or Diviner gets updated,
	// not just if the file content changes. Derive a hash from everything
	// we care about.
	$file_hashes[$rel_file] = md5("{$rel_file}\0{$md5_hash}\0{$version}").'F';
	}

	return $file_hashes;
	}

	private function deleteDeadAtoms(array $file_hashes) {
	$atom_cache = $this->getAtomCache();

	$hash_to_file = array_flip($file_hashes);
	foreach ($atom_cache->getFileHashMap() as $hash => $atom) {
	if (empty($hash_to_file[$hash])) {
	$atom_cache->deleteFileHash($hash);
	}
	}
	}

	private function getFilesToAtomize(array $file_hashes) {
	$atom_cache = $this->getAtomCache();

	$atomize = array();
	foreach ($file_hashes as $file => $hash) {
	if (!$atom_cache->fileHashExists($hash)) {
	$atomize[] = $file;
	}
	}

	return $atomize;
	}

	private function buildAtomizerFutures(array $file_atomizers) {
	$atomizers = array();
	foreach ($file_atomizers as $file => $atomizer) {
	$atomizers[$atomizer][] = $file;
	}

	$root = dirname(phutil_get_library_root('phabricator'));
	$config_root = $this->getConfig('root');

	$bar = id(new PhutilConsoleProgressBar())
	->setTotal(count($file_atomizers));

	$futures = array();
	foreach ($atomizers as $class => $files) {
	foreach (array_chunk($files, 32) as $chunk) {
	$future = new ExecFuture(
	'%s atomize --ugly --book %s --atomizer %s -- %Ls',
	$root.'/bin/diviner',
	$this->getBookConfigPath(),
	$class,
	$chunk);
	$future->setCWD($config_root);

	$futures[] = $future;

	$bar->update(count($chunk));
	}
	}

	$bar->done();

	return $futures;
	}

	private function resolveAtomizerFutures(array $futures, array $file_hashes) {
	assert_instances_of($futures, 'Future');

	$atom_cache = $this->getAtomCache();
	$bar = id(new PhutilConsoleProgressBar())
	->setTotal(count($futures));
	- foreach (Futures($futures)->limit(4) as $key => $future) {
	+ $futures = id(new FutureIterator($futures))
	+ ->limit(4);
	+ foreach ($futures as $key => $future) {
	try {
	$atoms = $future->resolveJSON();

	foreach ($atoms as $atom) {
	if ($atom['type'] == DivinerAtom::TYPE_FILE) {
	$file_hash = $file_hashes[$atom['file']];
	$atom_cache->addFileHash($file_hash, $atom['hash']);
	}
	$atom_cache->addAtom($atom);
	}
	} catch (Exception $e) {
	phlog($e);
	}

	$bar->update(1);
	}
	$bar->done();
	}


	/**
	* Get a global version number, which changes whenever any atom or atomizer
	* implementation changes in a way which is not backward-compatible.
	*/
	private function getDivinerAtomWorldVersion() {
	$version = array();
	$version['atom'] = DivinerAtom::getAtomSerializationVersion();
	$version['rules'] = $this->getRules();

	$atomizers = id(new PhutilSymbolLoader())
	->setAncestorClass('DivinerAtomizer')
	->setConcreteOnly(true)
	->selectAndLoadSymbols();

	$atomizer_versions = array();
	foreach ($atomizers as $atomizer) {
	$atomizer_versions[$atomizer['name']] = call_user_func(
	array(
	$atomizer['name'],
	'getAtomizerVersion',
	));
	}

	ksort($atomizer_versions);
	$version['atomizers'] = $atomizer_versions;

	return md5(serialize($version));
	}


	/* -( Graph Cache )-------------------------------------------------------- */


	private function buildGraphCache() {
	$this->log(pht('BUILDING GRAPH CACHE'));

	$atom_cache = $this->getAtomCache();
	$symbol_map = $atom_cache->getSymbolMap();
	$atoms = $atom_cache->getAtomMap();

	$dirty_symbols = array();
	$dirty_nhashes = array();

	$del_atoms = array_diff_key($symbol_map, $atoms);
	$this->log(pht('Found %d obsolete atom(s) in graph.', count($del_atoms)));
	foreach ($del_atoms as $nhash => $shash) {
	$atom_cache->deleteSymbol($nhash);
	$dirty_symbols[$shash] = true;

	$atom_cache->deleteEdges($nhash);
	$atom_cache->deleteGraph($nhash);
	}

	$new_atoms = array_diff_key($atoms, $symbol_map);
	$this->log(pht('Found %d new atom(s) in graph.', count($new_atoms)));
	foreach ($new_atoms as $nhash => $ignored) {
	$shash = $this->computeSymbolHash($nhash);
	$atom_cache->addSymbol($nhash, $shash);
	$dirty_symbols[$shash] = true;

	$atom_cache->addEdges(
	$nhash,
	$this->getEdges($nhash));

	$dirty_nhashes[$nhash] = true;
	}

	$this->log(pht('Propagating changes through the graph.'));

	// Find all the nodes which point at a dirty node, and dirty them. Then
	// find all the nodes which point at those nodes and dirty them, and so
	// on. (This is slightly overkill since we probably don't need to propagate
	// dirtiness across documentation "links" between symbols, but we do want
	// to propagate it across "extends", and we suffer only a little bit of
	// collateral damage by over-dirtying as long as the documentation isn't
	// too well-connected.)

	$symbol_stack = array_keys($dirty_symbols);
	while ($symbol_stack) {
	$symbol_hash = array_pop($symbol_stack);

	foreach ($atom_cache->getEdgesWithDestination($symbol_hash) as $edge) {
	$dirty_nhashes[$edge] = true;
	$src_hash = $this->computeSymbolHash($edge);
	if (empty($dirty_symbols[$src_hash])) {
	$dirty_symbols[$src_hash] = true;
	$symbol_stack[] = $src_hash;
	}
	}
	}

	$this->log(pht('Found %d affected atoms.', count($dirty_nhashes)));

	foreach ($dirty_nhashes as $nhash => $ignored) {
	$atom_cache->addGraph($nhash, $this->computeGraphHash($nhash));
	}

	$this->log(pht('Writing graph cache.'));

	$atom_cache->saveGraph();
	$atom_cache->saveEdges();
	$atom_cache->saveSymbols();

	$this->log(pht('Done.')."\n");
	}

	private function computeSymbolHash($node_hash) {
	$atom_cache = $this->getAtomCache();
	$atom = $atom_cache->getAtom($node_hash);

	if (!$atom) {
	throw new Exception("No such atom with node hash '{$node_hash}'!");
	}

	$ref = DivinerAtomRef::newFromDictionary($atom['ref']);
	return $ref->toHash();
	}

	private function getEdges($node_hash) {
	$atom_cache = $this->getAtomCache();
	$atom = $atom_cache->getAtom($node_hash);

	$refs = array();

	// Make the atom depend on its own symbol, so that all atoms with the same
	// symbol are dirtied (e.g., if a codebase defines the function "f()"
	// several times, all of them should be dirtied when one is dirtied).
	$refs[DivinerAtomRef::newFromDictionary($atom)->toHash()] = true;

	foreach (array_merge($atom['extends'], $atom['links']) as $ref_dict) {
	$ref = DivinerAtomRef::newFromDictionary($ref_dict);
	if ($ref->getBook() == $atom['book']) {
	$refs[$ref->toHash()] = true;
	}
	}

	return array_keys($refs);
	}

	private function computeGraphHash($node_hash) {
	$atom_cache = $this->getAtomCache();
	$atom = $atom_cache->getAtom($node_hash);

	$edges = $this->getEdges($node_hash);
	sort($edges);

	$inputs = array(
	'atomHash' => $atom['hash'],
	'edges' => $edges,
	);

	return md5(serialize($inputs)).'G';
	}


	private function publishDocumentation($clean) {
	$atom_cache = $this->getAtomCache();
	$graph_map = $atom_cache->getGraphMap();

	$this->log(pht('PUBLISHING DOCUMENTATION'));

	$publisher = new DivinerLivePublisher();
	$publisher->setDropCaches($clean);
	$publisher->setConfig($this->getAllConfig());
	$publisher->setAtomCache($atom_cache);
	$publisher->setRenderer(new DivinerDefaultRenderer());
	$publisher->publishAtoms(array_values($graph_map));

	$this->log(pht('Done.'));
	}


	}
	diff --git a/src/applications/doorkeeper/bridge/DoorkeeperBridgeAsana.php b/src/applications/doorkeeper/bridge/DoorkeeperBridgeAsana.php
	index 79e711e00a..4e32239793 100644
	--- a/src/applications/doorkeeper/bridge/DoorkeeperBridgeAsana.php
	+++ b/src/applications/doorkeeper/bridge/DoorkeeperBridgeAsana.php
	@@ -1,129 +1,129 @@
	<?php

	final class DoorkeeperBridgeAsana extends DoorkeeperBridge {

	const APPTYPE_ASANA = 'asana';
	const APPDOMAIN_ASANA = 'asana.com';
	const OBJTYPE_TASK = 'asana:task';

	public function canPullRef(DoorkeeperObjectRef $ref) {
	if ($ref->getApplicationType() != self::APPTYPE_ASANA) {
	return false;
	}

	if ($ref->getApplicationDomain() != self::APPDOMAIN_ASANA) {
	return false;
	}

	$types = array(
	self::OBJTYPE_TASK => true,
	);

	return isset($types[$ref->getObjectType()]);
	}

	public function pullRefs(array $refs) {

	$id_map = mpull($refs, 'getObjectID', 'getObjectKey');
	$viewer = $this->getViewer();

	$provider = PhabricatorAsanaAuthProvider::getAsanaProvider();
	if (!$provider) {
	return;
	}

	$accounts = id(new PhabricatorExternalAccountQuery())
	->setViewer($viewer)
	->withUserPHIDs(array($viewer->getPHID()))
	->withAccountTypes(array($provider->getProviderType()))
	->withAccountDomains(array($provider->getProviderDomain()))
	->requireCapabilities(
	array(
	PhabricatorPolicyCapability::CAN_VIEW,
	PhabricatorPolicyCapability::CAN_EDIT,
	))
	->execute();

	if (!$accounts) {
	return $this->didFailOnMissingLink();
	}

	// TODO: If the user has several linked Asana accounts, we just pick the
	// first one arbitrarily. We might want to try using all of them or do
	// something with more finesse. There's no UI way to link multiple accounts
	// right now so this is currently moot.
	$account = head($accounts);

	$token = $provider->getOAuthAccessToken($account);
	if (!$token) {
	return;
	}

	$template = id(new PhutilAsanaFuture())
	->setAccessToken($token);

	$futures = array();
	foreach ($id_map as $key => $id) {
	$futures[$key] = id(clone $template)
	->setRawAsanaQuery("tasks/{$id}");
	}

	$results = array();
	$failed = array();
	- foreach (Futures($futures) as $key => $future) {
	+ foreach (new FutureIterator($futures) as $key => $future) {
	try {
	$results[$key] = $future->resolve();
	} catch (Exception $ex) {
	if (($ex instanceof HTTPFutureResponseStatus) &&
	($ex->getStatusCode() == 404)) {
	// This indicates that the object has been deleted (or never existed,
	// or isn't visible to the current user) but it's a successful sync of
	// an object which isn't visible.
	} else {
	// This is something else, so consider it a synchronization failure.
	phlog($ex);
	$failed[$key] = $ex;
	}
	}
	}

	foreach ($refs as $ref) {
	$ref->setAttribute('name', pht('Asana Task %s', $ref->getObjectID()));

	$did_fail = idx($failed, $ref->getObjectKey());
	if ($did_fail) {
	$ref->setSyncFailed(true);
	continue;
	}

	$result = idx($results, $ref->getObjectKey());
	if (!$result) {
	continue;
	}

	$ref->setIsVisible(true);
	$ref->setAttribute('asana.data', $result);
	$ref->setAttribute('fullname', pht('Asana: %s', $result['name']));
	$ref->setAttribute('title', $result['name']);
	$ref->setAttribute('description', $result['notes']);

	$obj = $ref->getExternalObject();
	if ($obj->getID()) {
	continue;
	}

	$this->fillObjectFromData($obj, $result);

	$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
	$obj->save();
	unset($unguarded);
	}
	}

	public function fillObjectFromData(DoorkeeperExternalObject $obj, $result) {
	$id = $result['id'];
	$uri = "https://app.asana.com/0/{$id}/{$id}";
	$obj->setObjectURI($uri);
	}

	}
	diff --git a/src/applications/doorkeeper/bridge/DoorkeeperBridgeJIRA.php b/src/applications/doorkeeper/bridge/DoorkeeperBridgeJIRA.php
	index 6349943770..c9b40d2e72 100644
	--- a/src/applications/doorkeeper/bridge/DoorkeeperBridgeJIRA.php
	+++ b/src/applications/doorkeeper/bridge/DoorkeeperBridgeJIRA.php
	@@ -1,148 +1,148 @@
	<?php

	final class DoorkeeperBridgeJIRA extends DoorkeeperBridge {

	const APPTYPE_JIRA = 'jira';
	const OBJTYPE_ISSUE = 'jira:issue';

	public function canPullRef(DoorkeeperObjectRef $ref) {
	if ($ref->getApplicationType() != self::APPTYPE_JIRA) {
	return false;
	}

	$types = array(
	self::OBJTYPE_ISSUE => true,
	);

	return isset($types[$ref->getObjectType()]);
	}

	public function pullRefs(array $refs) {

	$id_map = mpull($refs, 'getObjectID', 'getObjectKey');
	$viewer = $this->getViewer();

	$provider = PhabricatorJIRAAuthProvider::getJIRAProvider();
	if (!$provider) {
	return;
	}

	$accounts = id(new PhabricatorExternalAccountQuery())
	->setViewer($viewer)
	->withUserPHIDs(array($viewer->getPHID()))
	->withAccountTypes(array($provider->getProviderType()))
	->requireCapabilities(
	array(
	PhabricatorPolicyCapability::CAN_VIEW,
	PhabricatorPolicyCapability::CAN_EDIT,
	))
	->execute();

	if (!$accounts) {
	return $this->didFailOnMissingLink();
	}

	// TODO: When we support multiple JIRA instances, we need to disambiguate
	// issues (perhaps with additional configuration) or cast a wide net
	// (by querying all instances). For now, just query the one instance.
	$account = head($accounts);

	$futures = array();
	foreach ($id_map as $key => $id) {
	$futures[$key] = $provider->newJIRAFuture(
	$account,
	'rest/api/2/issue/'.phutil_escape_uri($id),
	'GET');
	}

	$results = array();
	$failed = array();
	- foreach (Futures($futures) as $key => $future) {
	+ foreach (new FutureIterator($futures) as $key => $future) {
	try {
	$results[$key] = $future->resolveJSON();
	} catch (Exception $ex) {
	if (($ex instanceof HTTPFutureResponseStatus) &&
	($ex->getStatusCode() == 404)) {
	// This indicates that the object has been deleted (or never existed,
	// or isn't visible to the current user) but it's a successful sync of
	// an object which isn't visible.
	} else {
	// This is something else, so consider it a synchronization failure.
	phlog($ex);
	$failed[$key] = $ex;
	}
	}
	}

	foreach ($refs as $ref) {
	$ref->setAttribute('name', pht('JIRA %s', $ref->getObjectID()));

	$did_fail = idx($failed, $ref->getObjectKey());
	if ($did_fail) {
	$ref->setSyncFailed(true);
	continue;
	}

	$result = idx($results, $ref->getObjectKey());
	if (!$result) {
	continue;
	}

	$fields = idx($result, 'fields', array());

	$ref->setIsVisible(true);
	$ref->setAttribute(
	'fullname',
	pht('JIRA %s %s', $result['key'], idx($fields, 'summary')));

	$ref->setAttribute('title', idx($fields, 'summary'));
	$ref->setAttribute('description', idx($result, 'description'));

	$obj = $ref->getExternalObject();
	if ($obj->getID()) {
	continue;
	}

	$this->fillObjectFromData($obj, $result);

	$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
	$obj->save();
	unset($unguarded);
	}
	}

	public function fillObjectFromData(DoorkeeperExternalObject $obj, $result) {
	// Convert the "self" URI, which points at the REST endpoint, into a
	// browse URI.
	$self = idx($result, 'self');
	$object_id = $obj->getObjectID();

	$uri = self::getJIRAIssueBrowseURIFromJIRARestURI($self, $object_id);
	if ($uri !== null) {
	$obj->setObjectURI($uri);
	}
	}

	public static function getJIRAIssueBrowseURIFromJIRARestURI(
	$uri,
	$object_id) {

	$uri = new PhutilURI($uri);

	// The JIRA install might not be at the domain root, so we may need to
	// keep an initial part of the path, like "/jira/". Find the API specific
	// part of the URI, strip it off, then replace it with the web version.
	$path = $uri->getPath();
	$pos = strrpos($path, 'rest/api/2/issue/');
	if ($pos === false) {
	return null;
	}

	$path = substr($path, 0, $pos);
	$path = $path.'browse/'.$object_id;
	$uri->setPath($path);

	return (string)$uri;
	}

	}
	diff --git a/src/applications/harbormaster/step/HarbormasterBuildStepImplementation.php b/src/applications/harbormaster/step/HarbormasterBuildStepImplementation.php
	index 3933cd5086..5057097fac 100644
	--- a/src/applications/harbormaster/step/HarbormasterBuildStepImplementation.php
	+++ b/src/applications/harbormaster/step/HarbormasterBuildStepImplementation.php
	@@ -1,250 +1,250 @@
	<?php

	abstract class HarbormasterBuildStepImplementation {

	public static function getImplementations() {
	return id(new PhutilSymbolLoader())
	->setAncestorClass('HarbormasterBuildStepImplementation')
	->loadObjects();
	}

	public static function getImplementation($class) {
	$base = idx(self::getImplementations(), $class);

	if ($base) {
	return (clone $base);
	}

	return null;
	}

	public static function requireImplementation($class) {
	if (!$class) {
	throw new Exception(pht('No implementation is specified!'));
	}

	$implementation = self::getImplementation($class);
	if (!$implementation) {
	throw new Exception(pht('No such implementation "%s" exists!', $class));
	}

	return $implementation;
	}

	/**
	* The name of the implementation.
	*/
	abstract public function getName();

	/**
	* The generic description of the implementation.
	*/
	public function getGenericDescription() {
	return '';
	}

	/**
	* The description of the implementation, based on the current settings.
	*/
	public function getDescription() {
	return $this->getGenericDescription();
	}

	/**
	* Run the build target against the specified build.
	*/
	abstract public function execute(
	HarbormasterBuild $build,
	HarbormasterBuildTarget $build_target);

	/**
	* Gets the settings for this build step.
	*/
	public function getSettings() {
	return $this->settings;
	}

	public function getSetting($key, $default = null) {
	return idx($this->settings, $key, $default);
	}

	/**
	* Loads the settings for this build step implementation from a build
	* step or target.
	*/
	public final function loadSettings($build_object) {
	$this->settings = $build_object->getDetails();
	return $this;
	}

	/**
	* Return the name of artifacts produced by this command.
	*
	* Something like:
	*
	* return array(
	* 'some_name_input_by_user' => 'host');
	*
	* Future steps will calculate all available artifact mappings
	* before them and filter on the type.
	*
	* @return array The mappings of artifact names to their types.
	*/
	public function getArtifactInputs() {
	return array();
	}

	public function getArtifactOutputs() {
	return array();
	}

	public function getDependencies(HarbormasterBuildStep $build_step) {
	$dependencies = $build_step->getDetail('dependsOn', array());

	$inputs = $build_step->getStepImplementation()->getArtifactInputs();
	$inputs = ipull($inputs, null, 'key');

	$artifacts = $this->getAvailableArtifacts(
	$build_step->getBuildPlan(),
	$build_step,
	null);

	foreach ($artifacts as $key => $type) {
	if (!array_key_exists($key, $inputs)) {
	unset($artifacts[$key]);
	}
	}

	$artifact_steps = ipull($artifacts, 'step');
	$artifact_steps = mpull($artifact_steps, 'getPHID');

	$dependencies = array_merge($dependencies, $artifact_steps);

	return $dependencies;
	}

	/**
	* Returns a list of all artifacts made available in the build plan.
	*/
	public static function getAvailableArtifacts(
	HarbormasterBuildPlan $build_plan,
	$current_build_step,
	$artifact_type) {

	$steps = id(new HarbormasterBuildStepQuery())
	->setViewer(PhabricatorUser::getOmnipotentUser())
	->withBuildPlanPHIDs(array($build_plan->getPHID()))
	->execute();

	$artifacts = array();

	$artifact_arrays = array();
	foreach ($steps as $step) {
	if ($current_build_step !== null &&
	$step->getPHID() === $current_build_step->getPHID()) {

	continue;
	}

	$implementation = $step->getStepImplementation();
	$array = $implementation->getArtifactOutputs();
	$array = ipull($array, 'type', 'key');
	foreach ($array as $name => $type) {
	if ($type !== $artifact_type && $artifact_type !== null) {
	continue;
	}
	$artifacts[$name] = array('type' => $type, 'step' => $step);
	}
	}

	return $artifacts;
	}

	/**
	* Convert a user-provided string with variables in it, like:
	*
	* ls ${dirname}
	*
	* ...into a string with variables merged into it safely:
	*
	* ls 'dir with spaces'
	*
	* @param string Name of a `vxsprintf` function, like @{function:vcsprintf}.
	* @param string User-provided pattern string containing `${variables}`.
	* @param dict List of available replacement variables.
	* @return string String with variables replaced safely into it.
	*/
	protected function mergeVariables($function, $pattern, array $variables) {
	$regexp = '/\\$\\{(?P<name>[a-z\\.]+)\\}/';

	$matches = null;
	preg_match_all($regexp, $pattern, $matches);

	$argv = array();
	foreach ($matches['name'] as $name) {
	if (!array_key_exists($name, $variables)) {
	throw new Exception(pht("No such variable '%s'!", $name));
	}
	$argv[] = $variables[$name];
	}

	$pattern = str_replace('%', '%%', $pattern);
	$pattern = preg_replace($regexp, '%s', $pattern);

	return call_user_func($function, $pattern, $argv);
	}

	public function getFieldSpecifications() {
	return array();
	}

	protected function formatSettingForDescription($key, $default = null) {
	return $this->formatValueForDescription($this->getSetting($key, $default));
	}

	protected function formatValueForDescription($value) {
	if (strlen($value)) {
	return phutil_tag('strong', array(), $value);
	} else {
	return phutil_tag('em', array(), pht('(null)'));
	}
	}

	public function supportsWaitForMessage() {
	return false;
	}

	public function shouldWaitForMessage(HarbormasterBuildTarget $target) {
	if (!$this->supportsWaitForMessage()) {
	return false;
	}

	return (bool)$target->getDetail('builtin.wait-for-message');
	}

	protected function shouldAbort(
	HarbormasterBuild $build,
	HarbormasterBuildTarget $target) {

	return $build->getBuildGeneration() !== $target->getBuildGeneration();
	}

	protected function resolveFuture(
	HarbormasterBuild $build,
	HarbormasterBuildTarget $target,
	Future $future) {

	- $futures = Futures(array($future));
	+ $futures = new FutureIterator(array($future));
	foreach ($futures->setUpdateInterval(5) as $key => $future) {
	if ($future === null) {
	$build->reload();
	if ($this->shouldAbort($build, $target)) {
	throw new HarbormasterBuildAbortedException();
	}
	} else {
	return $future->resolve();
	}
	}
	}

	}
	diff --git a/src/applications/harbormaster/step/HarbormasterCommandBuildStepImplementation.php b/src/applications/harbormaster/step/HarbormasterCommandBuildStepImplementation.php
	index 806e311083..14fc65a76c 100644
	--- a/src/applications/harbormaster/step/HarbormasterCommandBuildStepImplementation.php
	+++ b/src/applications/harbormaster/step/HarbormasterCommandBuildStepImplementation.php
	@@ -1,144 +1,144 @@
	<?php

	final class HarbormasterCommandBuildStepImplementation
	extends HarbormasterBuildStepImplementation {

	private $platform;

	public function getName() {
	return pht('Run Command');
	}

	public function getGenericDescription() {
	return pht('Run a command on Drydock host.');
	}

	public function getDescription() {
	return pht(
	'Run command %s on host %s.',
	$this->formatSettingForDescription('command'),
	$this->formatSettingForDescription('hostartifact'));
	}

	public function escapeCommand($pattern, array $args) {
	array_unshift($args, $pattern);

	$mode = PhutilCommandString::MODE_DEFAULT;
	if ($this->platform == 'windows') {
	$mode = PhutilCommandString::MODE_POWERSHELL;
	}

	return id(new PhutilCommandString($args))
	->setEscapingMode($mode);
	}

	public function execute(
	HarbormasterBuild $build,
	HarbormasterBuildTarget $build_target) {

	$settings = $this->getSettings();
	$variables = $build_target->getVariables();

	$artifact = $build->loadArtifact($settings['hostartifact']);

	$lease = $artifact->loadDrydockLease();

	$this->platform = $lease->getAttribute('platform');

	$command = $this->mergeVariables(
	array($this, 'escapeCommand'),
	$settings['command'],
	$variables);

	$this->platform = null;

	$interface = $lease->getInterface('command');

	$future = $interface->getExecFuture('%C', $command);

	$log_stdout = $build->createLog($build_target, 'remote', 'stdout');
	$log_stderr = $build->createLog($build_target, 'remote', 'stderr');

	$start_stdout = $log_stdout->start();
	$start_stderr = $log_stderr->start();

	$build_update = 5;

	// Read the next amount of available output every second.
	- $futures = Futures(array($future));
	+ $futures = new FutureIterator(array($future));
	foreach ($futures->setUpdateInterval(1) as $key => $future_iter) {
	if ($future_iter === null) {

	// Check to see if we should abort.
	if ($build_update <= 0) {
	$build->reload();
	if ($this->shouldAbort($build, $build_target)) {
	$future->resolveKill();
	throw new HarbormasterBuildAbortedException();
	} else {
	$build_update = 5;
	}
	} else {
	$build_update -= 1;
	}

	// Command is still executing.

	// Read more data as it is available.
	list($stdout, $stderr) = $future->read();
	$log_stdout->append($stdout);
	$log_stderr->append($stderr);
	$future->discardBuffers();
	} else {
	// Command execution is complete.

	// Get the return value so we can log that as well.
	list($err) = $future->resolve();

	// Retrieve the last few bits of information.
	list($stdout, $stderr) = $future->read();
	$log_stdout->append($stdout);
	$log_stderr->append($stderr);
	$future->discardBuffers();

	break;
	}
	}

	$log_stdout->finalize($start_stdout);
	$log_stderr->finalize($start_stderr);

	if ($err) {
	throw new HarbormasterBuildFailureException();
	}
	}

	public function getArtifactInputs() {
	return array(
	array(
	'name' => pht('Run on Host'),
	'key' => $this->getSetting('hostartifact'),
	'type' => HarbormasterBuildArtifact::TYPE_HOST,
	),
	);
	}

	public function getFieldSpecifications() {
	return array(
	'command' => array(
	'name' => pht('Command'),
	'type' => 'text',
	'required' => true,
	'caption' => pht(
	'Under Windows, this is executed under PowerShell.'.
	'Under UNIX, this is executed using the user\'s shell.'),
	),
	'hostartifact' => array(
	'name' => pht('Host'),
	'type' => 'text',
	'required' => true,
	),
	);
	}

	}
	diff --git a/src/infrastructure/internationalization/management/PhabricatorInternationalizationManagementExtractWorkflow.php b/src/infrastructure/internationalization/management/PhabricatorInternationalizationManagementExtractWorkflow.php
	index 6aca557188..e2819d1f28 100644
	--- a/src/infrastructure/internationalization/management/PhabricatorInternationalizationManagementExtractWorkflow.php
	+++ b/src/infrastructure/internationalization/management/PhabricatorInternationalizationManagementExtractWorkflow.php
	@@ -1,99 +1,102 @@
	<?php

	final class PhabricatorInternationalizationManagementExtractWorkflow
	extends PhabricatorInternationalizationManagementWorkflow {

	public function didConstruct() {
	$this
	->setName('extract')
	->setSynopsis(pht('Extract translatable strings.'))
	->setArguments(
	array(
	array(
	'name' => 'paths',
	'wildcard' => true,
	),
	));
	}

	public function execute(PhutilArgumentParser $args) {
	$console = PhutilConsole::getConsole();
	$paths = $args->getArg('paths');

	$futures = array();
	foreach ($paths as $path) {
	$root = Filesystem::resolvePath($path);
	$path_files = id(new FileFinder($root))
	->withType('f')
	->withSuffix('php')
	->find();

	foreach ($path_files as $file) {
	$full_path = $root.DIRECTORY_SEPARATOR.$file;
	$data = Filesystem::readFile($full_path);
	$futures[$full_path] = xhpast_get_parser_future($data);
	}
	}

	$console->writeOut(
	"%s\n",
	pht('Found %s file(s)...', new PhutilNumber(count($futures))));

	$results = array();

	$bar = id(new PhutilConsoleProgressBar())
	->setTotal(count($futures));
	- foreach (Futures($futures)->limit(8) as $full_path => $future) {
	+
	+ $futures = id(new FutureIterator($futures))
	+ ->limit(8);
	+ foreach ($futures as $full_path => $future) {
	$bar->update(1);

	$tree = XHPASTTree::newFromDataAndResolvedExecFuture(
	Filesystem::readFile($full_path),
	$future->resolve());

	$root = $tree->getRootNode();
	$calls = $root->selectDescendantsOfType('n_FUNCTION_CALL');
	foreach ($calls as $call) {
	$name = $call->getChildByIndex(0)->getConcreteString();
	if ($name == 'pht') {
	$params = $call->getChildByIndex(1, 'n_CALL_PARAMETER_LIST');
	$string_node = $params->getChildByIndex(0);
	$string_line = $string_node->getLineNumber();
	try {
	$string_value = $string_node->evalStatic();

	$results[$string_value][] = array(
	'file' => Filesystem::readablePath($full_path),
	'line' => $string_line,
	);
	} catch (Exception $ex) {
	// TODO: Deal with this junks.
	}
	}
	}

	$tree->dispose();
	}
	$bar->done();

	ksort($results);

	$out = array();
	$out[] = '<?php';
	$out[] = '// @nolint';
	$out[] = 'return array(';
	foreach ($results as $string => $locations) {
	foreach ($locations as $location) {
	$out[] = ' // '.$location['file'].':'.$location['line'];
	}
	$out[] = " '".addcslashes($string, "\0..\37\\'\177..\377")."' => null,";
	$out[] = null;
	}
	$out[] = ');';
	$out[] = null;

	echo implode("\n", $out);

	return 0;
	}

	}
	diff --git a/src/infrastructure/lint/linter/PhabricatorJavelinLinter.php b/src/infrastructure/lint/linter/PhabricatorJavelinLinter.php
	index c69d8aacbc..4a2298a72a 100644
	--- a/src/infrastructure/lint/linter/PhabricatorJavelinLinter.php
	+++ b/src/infrastructure/lint/linter/PhabricatorJavelinLinter.php
	@@ -1,280 +1,280 @@
	<?php

	final class PhabricatorJavelinLinter extends ArcanistLinter {

	private $symbols = array();

	private $symbolsBinary;
	private $haveWarnedAboutBinary;

	const LINT_PRIVATE_ACCESS = 1;
	const LINT_MISSING_DEPENDENCY = 2;
	const LINT_UNNECESSARY_DEPENDENCY = 3;
	const LINT_UNKNOWN_DEPENDENCY = 4;
	const LINT_MISSING_BINARY = 5;

	public function getInfoName() {
	return 'Javelin Linter';
	}

	public function getInfoDescription() {
	return pht(
	'This linter is intended for use with the Javelin JS library and '.
	'extensions. Use `javelinsymbols` to run Javelin rules on Javascript '.
	'source files.');
	}

	private function getBinaryPath() {
	if ($this->symbolsBinary === null) {
	list($err, $stdout) = exec_manual('which javelinsymbols');
	$this->symbolsBinary = ($err ? false : rtrim($stdout));
	}
	return $this->symbolsBinary;
	}

	public function willLintPaths(array $paths) {
	if (!$this->getBinaryPath()) {
	return;
	}

	$root = dirname(phutil_get_library_root('phabricator'));
	require_once $root.'/scripts/__init_script__.php';

	$futures = array();
	foreach ($paths as $path) {
	if ($this->shouldIgnorePath($path)) {
	continue;
	}

	$future = $this->newSymbolsFuture($path);
	$futures[$path] = $future;
	}

	- foreach (Futures($futures)->limit(8) as $path => $future) {
	+ foreach (id(new FutureIterator($futures))->limit(8) as $path => $future) {
	$this->symbols[$path] = $future->resolvex();
	}
	}

	public function getLinterName() {
	return 'JAVELIN';
	}

	public function getLinterConfigurationName() {
	return 'javelin';
	}

	public function getLintSeverityMap() {
	return array(
	self::LINT_MISSING_BINARY => ArcanistLintSeverity::SEVERITY_WARNING,
	);
	}

	public function getLintNameMap() {
	return array(
	self::LINT_PRIVATE_ACCESS => 'Private Method/Member Access',
	self::LINT_MISSING_DEPENDENCY => 'Missing Javelin Dependency',
	self::LINT_UNNECESSARY_DEPENDENCY => 'Unnecessary Javelin Dependency',
	self::LINT_UNKNOWN_DEPENDENCY => 'Unknown Javelin Dependency',
	self::LINT_MISSING_BINARY => '`javelinsymbols` Not In Path',
	);
	}

	public function getCacheGranularity() {
	return ArcanistLinter::GRANULARITY_REPOSITORY;
	}

	public function getCacheVersion() {
	$version = '0';
	$binary_path = $this->getBinaryPath();
	if ($binary_path) {
	$version .= '-'.md5_file($binary_path);
	}
	return $version;
	}

	private function shouldIgnorePath($path) {
	return preg_match('@/__tests__/\|externals/javelin/docs/@', $path);
	}

	public function lintPath($path) {
	if ($this->shouldIgnorePath($path)) {
	return;
	}

	if (!$this->symbolsBinary) {
	if (!$this->haveWarnedAboutBinary) {
	$this->haveWarnedAboutBinary = true;
	// TODO: Write build documentation for the Javelin binaries and point
	// the user at it.
	$this->raiseLintAtLine(
	1,
	0,
	self::LINT_MISSING_BINARY,
	"The 'javelinsymbols' binary in the Javelin project is not ".
	"available in \$PATH, so the Javelin linter can't run. This ".
	"isn't a big concern, but means some Javelin problems can't be ".
	"automatically detected.");
	}
	return;
	}

	list($uses, $installs) = $this->getUsedAndInstalledSymbolsForPath($path);
	foreach ($uses as $symbol => $line) {
	$parts = explode('.', $symbol);
	foreach ($parts as $part) {
	if ($part[0] == '_' && $part[1] != '_') {
	$base = implode('.', array_slice($parts, 0, 2));
	if (!array_key_exists($base, $installs)) {
	$this->raiseLintAtLine(
	$line,
	0,
	self::LINT_PRIVATE_ACCESS,
	"This file accesses private symbol '{$symbol}' across file ".
	"boundaries. You may only access private members and methods ".
	"from the file where they are defined.");
	}
	break;
	}
	}
	}

	if ($this->getEngine()->getCommitHookMode()) {
	// Don't do the dependency checks in commit-hook mode because we won't
	// have an available working copy.
	return;
	}

	$external_classes = array();
	foreach ($uses as $symbol => $line) {
	$parts = explode('.', $symbol);
	$class = implode('.', array_slice($parts, 0, 2));
	if (!array_key_exists($class, $external_classes) &&
	!array_key_exists($class, $installs)) {
	$external_classes[$class] = $line;
	}
	}

	$celerity = CelerityResourceMap::getNamedInstance('phabricator');

	$path = preg_replace(
	'@^externals/javelinjs/src/@',
	'webroot/rsrc/js/javelin/',
	$path);
	$need = $external_classes;

	$resource_name = substr($path, strlen('webroot/'));
	$requires = $celerity->getRequiredSymbolsForName($resource_name);
	if (!$requires) {
	$requires = array();
	}

	foreach ($requires as $key => $requires_symbol) {
	$requires_name = $celerity->getResourceNameForSymbol($requires_symbol);
	if ($requires_name === null) {
	$this->raiseLintAtLine(
	0,
	0,
	self::LINT_UNKNOWN_DEPENDENCY,
	"This file @requires component '{$requires_symbol}', but it does ".
	"not exist. You may need to rebuild the Celerity map.");
	unset($requires[$key]);
	continue;
	}

	if (preg_match('/\\.css$/', $requires_name)) {
	// If JS requires CSS, just assume everything is fine.
	unset($requires[$key]);
	} else {
	$symbol_path = 'webroot/'.$requires_name;
	list($ignored, $req_install) = $this->getUsedAndInstalledSymbolsForPath(
	$symbol_path);
	if (array_intersect_key($req_install, $external_classes)) {
	$need = array_diff_key($need, $req_install);
	unset($requires[$key]);
	}
	}
	}

	foreach ($need as $class => $line) {
	$this->raiseLintAtLine(
	$line,
	0,
	self::LINT_MISSING_DEPENDENCY,
	"This file uses '{$class}' but does not @requires the component ".
	"which installs it. You may need to rebuild the Celerity map.");
	}

	foreach ($requires as $component) {
	$this->raiseLintAtLine(
	0,
	0,
	self::LINT_UNNECESSARY_DEPENDENCY,
	"This file @requires component '{$component}' but does not use ".
	"anything it provides.");
	}
	}

	private function loadSymbols($path) {
	if (empty($this->symbols[$path])) {
	$this->symbols[$path] = $this->newSymbolsFuture($path)->resolvex();
	}
	return $this->symbols[$path];
	}

	private function newSymbolsFuture($path) {
	$future = new ExecFuture('javelinsymbols # %s', $path);
	$future->write($this->getData($path));
	return $future;
	}

	private function getUsedAndInstalledSymbolsForPath($path) {
	list($symbols) = $this->loadSymbols($path);
	$symbols = trim($symbols);

	$uses = array();
	$installs = array();
	if (empty($symbols)) {
	// This file has no symbols.
	return array($uses, $installs);
	}

	$symbols = explode("\n", trim($symbols));
	foreach ($symbols as $line) {
	$matches = null;
	if (!preg_match('/^([?+\])([^:]):(\d+)$/', $line, $matches)) {
	throw new Exception(
	'Received malformed output from `javelinsymbols`.');
	}
	$type = $matches[1];
	$symbol = $matches[2];
	$line = $matches[3];

	switch ($type) {
	case '?':
	$uses[$symbol] = $line;
	break;
	case '+':
	$installs['JX.'.$symbol] = $line;
	break;
	}
	}

	$contents = $this->getData($path);

	$matches = null;
	$count = preg_match_all(
	'/@javelin-installs\W+(\S+)/',
	$contents,
	$matches,
	PREG_PATTERN_ORDER);

	if ($count) {
	foreach ($matches[1] as $symbol) {
	$installs[$symbol] = 0;
	}
	}

	return array($uses, $installs);
	}

	}

File Metadata

Mime Type: text/x-diff
Expires: Mon, Mar 16, 10:29 PM (17 h, 42 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 963455
Default Alt Text: (171 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions