No OneTemporary
Actions

Size

33 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
	index f4f8da7550..2b3845b136 100644
	--- a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
	+++ b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
	@@ -1,436 +1,334 @@
	<?php

	/**
	* Run pull commands on local working copies to keep them up to date. This
	* daemon handles all repository types.
	*
	* By default, the daemon pulls every repository. If you want it to be
	* responsible for only some repositories, you can launch it with a list of
	* PHIDs or callsigns:
	*
	* ./phd launch repositorypulllocal -- X Q Z
	*
	* You can also launch a daemon which is responsible for all //but// one or
	* more repositories:
	*
	* ./phd launch repositorypulllocal -- --not A --not B
	*
	* If you have a very large number of repositories and some aren't being pulled
	* as frequently as you'd like, you can either change the pull frequency of
	* the less-important repositories to a larger number (so the daemon will skip
	* them more often) or launch one daemon for all the less-important repositories
	* and one for the more important repositories (or one for each more important
	* repository).
	*
	* @task pull Pulling Repositories
	* @task git Git Implementation
	* @task hg Mercurial Implementation
	*/
	final class PhabricatorRepositoryPullLocalDaemon
	extends PhabricatorDaemon {

	- private $repair;
	private $discoveryEngines = array();

	- public function setRepair($repair) {
	- $this->repair = $repair;
	- return $this;
	- }
	-

	/* -( Pulling Repositories )----------------------------------------------- */


	/**
	* @task pull
	*/
	public function run() {
	$argv = $this->getArgv();
	array_unshift($argv, __CLASS__);
	$args = new PhutilArgumentParser($argv);
	$args->parse(
	array(
	array(
	'name' => 'no-discovery',
	'help' => 'Pull only, without discovering commits.',
	),
	array(
	'name' => 'not',
	'param' => 'repository',
	'repeat' => true,
	'help' => 'Do not pull __repository__.',
	),
	array(
	'name' => 'repositories',
	'wildcard' => true,
	'help' => 'Pull specific __repositories__ instead of all.',
	),
	));

	$no_discovery = $args->getArg('no-discovery');
	$repo_names = $args->getArg('repositories');
	$exclude_names = $args->getArg('not');

	// Each repository has an individual pull frequency; after we pull it,
	// wait that long to pull it again. When we start up, try to pull everything
	// serially.
	$retry_after = array();

	$min_sleep = 15;

	while (true) {
	$repositories = $this->loadRepositories($repo_names);
	if ($exclude_names) {
	$exclude = $this->loadRepositories($exclude_names);
	$repositories = array_diff_key($repositories, $exclude);
	}

	// Shuffle the repositories, then re-key the array since shuffle()
	// discards keys. This is mostly for startup, we'll use soft priorities
	// later.
	shuffle($repositories);
	$repositories = mpull($repositories, null, 'getID');

	// If any repositories have the NEEDS_UPDATE flag set, pull them
	// as soon as possible.
	$type_need_update = PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE;
	$need_update_messages = id(new PhabricatorRepositoryStatusMessage())
	->loadAllWhere('statusType = %s', $type_need_update);
	foreach ($need_update_messages as $message) {
	$retry_after[$message->getRepositoryID()] = time();
	}

	// If any repositories were deleted, remove them from the retry timer map
	// so we don't end up with a retry timer that never gets updated and
	// causes us to sleep for the minimum amount of time.
	$retry_after = array_select_keys(
	$retry_after,
	array_keys($repositories));

	// Assign soft priorities to repositories based on how frequently they
	// should pull again.
	asort($retry_after);
	$repositories = array_select_keys(
	$repositories,
	array_keys($retry_after)) + $repositories;

	foreach ($repositories as $id => $repository) {
	$after = idx($retry_after, $id, 0);
	if ($after > time()) {
	continue;
	}

	$tracked = $repository->isTracked();
	if (!$tracked) {
	continue;
	}

	$callsign = $repository->getCallsign();

	try {
	$this->log("Updating repository '{$callsign}'.");

	id(new PhabricatorRepositoryPullEngine())
	->setRepository($repository)
	->pullRepository();

	if (!$no_discovery) {
	// TODO: It would be nice to discover only if we pulled something,
	// but this isn't totally trivial. It's slightly more complicated
	// with hosted repositories, too.

	$lock_name = get_class($this).':'.$callsign;
	$lock = PhabricatorGlobalLock::newLock($lock_name);
	$lock->lock();

	try {
	$repository->writeStatusMessage(
	PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE,
	null);
	$this->discoverRepository($repository);
	$this->updateRepositoryRefs($repository);
	$repository->writeStatusMessage(
	PhabricatorRepositoryStatusMessage::TYPE_FETCH,
	PhabricatorRepositoryStatusMessage::CODE_OKAY);
	} catch (Exception $ex) {
	$repository->writeStatusMessage(
	PhabricatorRepositoryStatusMessage::TYPE_FETCH,
	PhabricatorRepositoryStatusMessage::CODE_ERROR,
	array(
	'message' => pht(
	'Error updating working copy: %s', $ex->getMessage()),
	));
	$lock->unlock();
	throw $ex;
	}

	$lock->unlock();
	}

	$sleep_for = $repository->getDetail('pull-frequency', $min_sleep);
	$retry_after[$id] = time() + $sleep_for;
	} catch (PhutilLockException $ex) {
	$retry_after[$id] = time() + $min_sleep;
	$this->log("Failed to acquire lock.");
	} catch (Exception $ex) {
	$retry_after[$id] = time() + $min_sleep;

	$proxy = new PhutilProxyException(
	"Error while fetching changes to the '{$callsign}' repository.",
	$ex);
	phlog($proxy);
	}

	$this->stillWorking();
	}

	if ($retry_after) {
	$sleep_until = max(min($retry_after), time() + $min_sleep);
	} else {
	$sleep_until = time() + $min_sleep;
	}

	$this->sleep($sleep_until - time());
	}
	}


	/**
	* @task pull
	*/
	protected function loadRepositories(array $names) {
	$query = id(new PhabricatorRepositoryQuery())
	->setViewer($this->getViewer());

	if ($names) {
	$query->withCallsigns($names);
	}

	$repos = $query->execute();

	if ($names) {
	$by_callsign = mpull($repos, null, 'getCallsign');
	foreach ($names as $name) {
	if (empty($by_callsign[$name])) {
	throw new Exception(
	"No repository exists with callsign '{$name}'!");
	}
	}
	}

	return $repos;
	}

	public function discoverRepository(PhabricatorRepository $repository) {
	$refs = $this->getDiscoveryEngine($repository)
	->discoverCommits();

	- foreach ($refs as $ref) {
	- $this->recordCommit(
	- $repository,
	- $ref->getIdentifier(),
	- $ref->getEpoch(),
	- $ref->getCanCloseImmediately());
	- }
	-
	$this->checkIfRepositoryIsFullyImported($repository);

	try {
	$this->pushToMirrors($repository);
	} catch (Exception $ex) {
	// TODO: We should report these into the UI properly, but for
	// now just complain. These errors are much less severe than
	// pull errors.
	$proxy = new PhutilProxyException(
	pht(
	'Error while pushing "%s" repository to a mirror.',
	$repository->getCallsign()),
	$ex);
	phlog($proxy);
	}

	return (bool)count($refs);
	}

	private function updateRepositoryRefs(PhabricatorRepository $repository) {
	id(new PhabricatorRepositoryRefEngine())
	->setRepository($repository)
	->updateRefs();
	}

	private function getDiscoveryEngine(PhabricatorRepository $repository) {
	$id = $repository->getID();
	if (empty($this->discoveryEngines[$id])) {
	$engine = id(new PhabricatorRepositoryDiscoveryEngine())
	- ->setRepository($repository)
	- ->setVerbose($this->getVerbose())
	- ->setRepairMode($this->repair);
	+ ->setRepository($repository)
	+ ->setVerbose($this->getVerbose());

	$this->discoveryEngines[$id] = $engine;
	}
	return $this->discoveryEngines[$id];
	}

	- private function recordCommit(
	- PhabricatorRepository $repository,
	- $commit_identifier,
	- $epoch,
	- $close_immediately) {
	-
	- $commit = new PhabricatorRepositoryCommit();
	- $commit->setRepositoryID($repository->getID());
	- $commit->setCommitIdentifier($commit_identifier);
	- $commit->setEpoch($epoch);
	- if ($close_immediately) {
	- $commit->setImportStatus(PhabricatorRepositoryCommit::IMPORTED_CLOSEABLE);
	- }
	-
	- $data = new PhabricatorRepositoryCommitData();
	-
	- try {
	- $commit->openTransaction();
	- $commit->save();
	- $data->setCommitID($commit->getID());
	- $data->save();
	- $commit->saveTransaction();
	-
	- $this->insertTask($repository, $commit);
	-
	- queryfx(
	- $repository->establishConnection('w'),
	- 'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
	- VALUES (%d, 1, %d, %d)
	- ON DUPLICATE KEY UPDATE
	- size = size + 1,
	- lastCommitID =
	- IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID),
	- epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)',
	- PhabricatorRepository::TABLE_SUMMARY,
	- $repository->getID(),
	- $commit->getID(),
	- $epoch);
	-
	- if ($this->repair) {
	- // Normally, the query should throw a duplicate key exception. If we
	- // reach this in repair mode, we've actually performed a repair.
	- $this->log("Repaired commit '{$commit_identifier}'.");
	- }
	-
	- PhutilEventEngine::dispatchEvent(
	- new PhabricatorEvent(
	- PhabricatorEventType::TYPE_DIFFUSION_DIDDISCOVERCOMMIT,
	- array(
	- 'repository' => $repository,
	- 'commit' => $commit,
	- )));
	-
	- } catch (AphrontQueryDuplicateKeyException $ex) {
	- $commit->killTransaction();
	- // Ignore. This can happen because we discover the same new commit
	- // more than once when looking at history, or because of races or
	- // data inconsistency or cosmic radiation; in any case, we're still
	- // in a good state if we ignore the failure.
	- }
	- }
	-
	- private function insertTask(
	- PhabricatorRepository $repository,
	- PhabricatorRepositoryCommit $commit,
	- $data = array()) {
	-
	- $vcs = $repository->getVersionControlSystem();
	- switch ($vcs) {
	- case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
	- $class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
	- break;
	- case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
	- $class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
	- break;
	- case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
	- $class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker';
	- break;
	- default:
	- throw new Exception("Unknown repository type '{$vcs}'!");
	- }
	-
	- $data['commitID'] = $commit->getID();
	-
	- PhabricatorWorker::scheduleTask($class, $data);
	- }
	-
	private function checkIfRepositoryIsFullyImported(
	PhabricatorRepository $repository) {

	// Check if the repository has the "Importing" flag set. We want to clear
	// the flag if we can.
	$importing = $repository->getDetail('importing');
	if (!$importing) {
	// This repository isn't marked as "Importing", so we're done.
	return;
	}

	// Look for any commit which hasn't imported.
	$unparsed_commit = queryfx_one(
	$repository->establishConnection('r'),
	'SELECT * FROM %T WHERE repositoryID = %d AND importStatus != %d
	LIMIT 1',
	id(new PhabricatorRepositoryCommit())->getTableName(),
	$repository->getID(),
	PhabricatorRepositoryCommit::IMPORTED_ALL);
	if ($unparsed_commit) {
	// We found a commit which still needs to import, so we can't clear the
	// flag.
	return;
	}

	// Clear the "importing" flag.
	$repository->openTransaction();
	$repository->beginReadLocking();
	$repository = $repository->reload();
	$repository->setDetail('importing', false);
	$repository->save();
	$repository->endReadLocking();
	$repository->saveTransaction();
	}


	private function pushToMirrors(PhabricatorRepository $repository) {
	if (!$repository->canMirror()) {
	return;
	}

	$mirrors = id(new PhabricatorRepositoryMirrorQuery())
	->setViewer($this->getViewer())
	->withRepositoryPHIDs(array($repository->getPHID()))
	->execute();

	// TODO: This is a little bit janky, but we don't have first-class
	// infrastructure for running remote commands against an arbitrary remote
	// right now. Just make an emphemeral copy of the repository and muck with
	// it a little bit. In the medium term, we should pull this command stuff
	// out and use it here and for "Land to ...".

	$proxy = clone $repository;
	$proxy->makeEphemeral();

	$proxy->setDetail('hosting-enabled', false);
	foreach ($mirrors as $mirror) {
	$proxy->setDetail('remote-uri', $mirror->getRemoteURI());
	$proxy->setCredentialPHID($mirror->getCredentialPHID());

	$this->log(pht('Pushing to remote "%s"...', $mirror->getRemoteURI()));

	if (!$proxy->isGit()) {
	throw new Exception('Unsupported VCS!');
	}

	$future = $proxy->getRemoteCommandFuture(
	'push --verbose --mirror -- %P',
	$proxy->getRemoteURIEnvelope());

	$future
	->setCWD($proxy->getLocalPath())
	->resolvex();
	}
	}
	}
	diff --git a/src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php b/src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
	index 77ba515688..b6f5fe4ba3 100644
	--- a/src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
	+++ b/src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
	@@ -1,456 +1,550 @@
	<?php

	/**
	* @task discover Discovering Repositories
	* @task svn Discovering Subversion Repositories
	* @task git Discovering Git Repositories
	* @task hg Discovering Mercurial Repositories
	* @task internal Internals
	*/
	final class PhabricatorRepositoryDiscoveryEngine
	extends PhabricatorRepositoryEngine {

	private $repairMode;
	private $commitCache = array();
	const MAX_COMMIT_CACHE_SIZE = 2048;


	/* -( Discovering Repositories )------------------------------------------- */


	public function setRepairMode($repair_mode) {
	$this->repairMode = $repair_mode;
	return $this;
	}


	public function getRepairMode() {
	return $this->repairMode;
	}


	/**
	* @task discovery
	*/
	public function discoverCommits() {
	$repository = $this->getRepository();

	$vcs = $repository->getVersionControlSystem();
	switch ($vcs) {
	case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
	$refs = $this->discoverSubversionCommits();
	break;
	case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
	$refs = $this->discoverMercurialCommits();
	break;
	case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
	$refs = $this->discoverGitCommits();
	break;
	default:
	throw new Exception("Unknown VCS '{$vcs}'!");
	}

	- // Mark discovered commits in the cache.
	+ // Record discovered commits and mark them in the cache.
	foreach ($refs as $ref) {
	+ $this->recordCommit(
	+ $repository,
	+ $ref->getIdentifier(),
	+ $ref->getEpoch(),
	+ $ref->getCanCloseImmediately());
	+
	$this->commitCache[$ref->getIdentifier()] = true;
	}

	return $refs;
	}


	/* -( Discovering Git Repositories )--------------------------------------- */


	/**
	* @task git
	*/
	private function discoverGitCommits() {
	$repository = $this->getRepository();

	if (!$repository->isHosted()) {
	$this->verifyGitOrigin($repository);
	}

	$branches = id(new DiffusionLowLevelGitRefQuery())
	->setRepository($repository)
	->withIsOriginBranch(true)
	->execute();

	if (!$branches) {
	// This repository has no branches at all, so we don't need to do
	// anything. Generally, this means the repository is empty.
	return array();
	}

	$branches = $this->sortBranches($branches);
	$branches = mpull($branches, 'getCommitIdentifier', 'getShortName');

	$this->log(
	pht(
	'Discovering commits in repository %s.',
	$repository->getCallsign()));

	$refs = array();
	foreach ($branches as $name => $commit) {
	$this->log(pht('Examining branch "%s", at "%s".', $name, $commit));

	if (!$repository->shouldTrackBranch($name)) {
	$this->log(pht("Skipping, branch is untracked."));
	continue;
	}

	if ($this->isKnownCommit($commit)) {
	$this->log(pht("Skipping, HEAD is known."));
	continue;
	}

	$this->log(pht("Looking for new commits."));

	$refs[] = $this->discoverStreamAncestry(
	new PhabricatorGitGraphStream($repository, $commit),
	$commit,
	$repository->shouldAutocloseBranch($name));
	}

	return array_mergev($refs);
	}


	/**
	* Verify that the "origin" remote exists, and points at the correct URI.
	*
	* This catches or corrects some types of misconfiguration, and also repairs
	* an issue where Git 1.7.1 does not create an "origin" for `--bare` clones.
	* See T4041.
	*
	* @param PhabricatorRepository Repository to verify.
	* @return void
	*/
	private function verifyGitOrigin(PhabricatorRepository $repository) {
	list($remotes) = $repository->execxLocalCommand(
	'remote show -n origin');

	$matches = null;
	if (!preg_match('/^\sFetch URL:\s(.?)\s$/m', $remotes, $matches)) {
	throw new Exception(
	"Expected 'Fetch URL' in 'git remote show -n origin'.");
	}

	$remote_uri = $matches[1];
	$expect_remote = $repository->getRemoteURI();

	if ($remote_uri == "origin") {
	// If a remote does not exist, git pretends it does and prints out a
	// made up remote where the URI is the same as the remote name. This is
	// definitely not correct.

	// Possibly, we should use `git remote --verbose` instead, which does not
	// suffer from this problem (but is a little more complicated to parse).
	$valid = false;
	$exists = false;
	} else {
	$normal_type_git = PhabricatorRepositoryURINormalizer::TYPE_GIT;

	$remote_normal = id(new PhabricatorRepositoryURINormalizer(
	$normal_type_git,
	$remote_uri))->getNormalizedPath();

	$expect_normal = id(new PhabricatorRepositoryURINormalizer(
	$normal_type_git,
	$expect_remote))->getNormalizedPath();

	$valid = ($remote_normal == $expect_normal);
	$exists = true;
	}

	if (!$valid) {
	if (!$exists) {
	// If there's no "origin" remote, just create it regardless of how
	// strongly we own the working copy. There is almost no conceivable
	// scenario in which this could do damage.
	$this->log(
	pht(
	'Remote "origin" does not exist. Creating "origin", with '.
	'URI "%s".',
	$expect_remote));
	$repository->execxLocalCommand(
	'remote add origin %P',
	$repository->getRemoteURIEnvelope());

	// NOTE: This doesn't fetch the origin (it just creates it), so we won't
	// know about origin branches until the next "pull" happens. That's fine
	// for our purposes, but might impact things in the future.
	} else {
	if ($repository->canDestroyWorkingCopy()) {
	// Bad remote, but we can try to repair it.
	$this->log(
	pht(
	'Remote "origin" exists, but is pointed at the wrong URI, "%s". '.
	'Resetting origin URI to "%s.',
	$remote_uri,
	$expect_remote));
	$repository->execxLocalCommand(
	'remote set-url origin %P',
	$repository->getRemoteURIEnvelope());
	} else {
	// Bad remote and we aren't comfortable repairing it.
	$message = pht(
	'Working copy at "%s" has a mismatched origin URI, "%s". '.
	'The expected origin URI is "%s". Fix your configuration, or '.
	'set the remote URI correctly. To avoid breaking anything, '.
	'Phabricator will not automatically fix this.',
	$repository->getLocalPath(),
	$remote_uri,
	$expect_remote);
	throw new Exception($message);
	}
	}
	}
	}


	/* -( Discovering Subversion Repositories )-------------------------------- */


	/**
	* @task svn
	*/
	private function discoverSubversionCommits() {
	$repository = $this->getRepository();

	$upper_bound = null;
	$limit = 1;
	$refs = array();
	do {
	// Find all the unknown commits on this path. Note that we permit
	// importing an SVN subdirectory rather than the entire repository, so
	// commits may be nonsequential.

	if ($upper_bound === null) {
	$at_rev = 'HEAD';
	} else {
	$at_rev = ($upper_bound - 1);
	}

	try {
	list($xml, $stderr) = $repository->execxRemoteCommand(
	'log --xml --quiet --limit %d %s',
	$limit,
	$repository->getSubversionBaseURI($at_rev));
	} catch (CommandException $ex) {
	$stderr = $ex->getStdErr();
	if (preg_match('/(path\|File) not found/', $stderr)) {
	// We've gone all the way back through history and this path was not
	// affected by earlier commits.
	break;
	}
	throw $ex;
	}

	$xml = phutil_utf8ize($xml);
	$log = new SimpleXMLElement($xml);
	foreach ($log->logentry as $entry) {
	$identifier = (int)$entry['revision'];
	$epoch = (int)strtotime((string)$entry->date[0]);
	$refs[$identifier] = id(new PhabricatorRepositoryCommitRef())
	->setIdentifier($identifier)
	->setEpoch($epoch)
	->setCanCloseImmediately(true);

	if ($upper_bound === null) {
	$upper_bound = $identifier;
	} else {
	$upper_bound = min($upper_bound, $identifier);
	}
	}

	// Discover 2, 4, 8, ... 256 logs at a time. This allows us to initially
	// import large repositories fairly quickly, while pulling only as much
	// data as we need in the common case (when we've already imported the
	// repository and are just grabbing one commit at a time).
	$limit = min($limit * 2, 256);

	} while ($upper_bound > 1 && !$this->isKnownCommit($upper_bound));

	krsort($refs);
	while ($refs && $this->isKnownCommit(last($refs)->getIdentifier())) {
	array_pop($refs);
	}
	$refs = array_reverse($refs);

	return $refs;
	}


	/* -( Discovering Mercurial Repositories )--------------------------------- */


	/**
	* @task hg
	*/
	private function discoverMercurialCommits() {
	$repository = $this->getRepository();

	$branches = id(new DiffusionLowLevelMercurialBranchesQuery())
	->setRepository($repository)
	->execute();

	$refs = array();
	foreach ($branches as $branch) {
	// NOTE: Mercurial branches may have multiple heads, so the names may
	// not be unique.
	$name = $branch->getShortName();
	$commit = $branch->getCommitIdentifier();

	$this->log(pht('Examining branch "%s" head "%s".', $name, $commit));
	if (!$repository->shouldTrackBranch($name)) {
	$this->log(pht("Skipping, branch is untracked."));
	continue;
	}

	if ($this->isKnownCommit($commit)) {
	$this->log(pht("Skipping, this head is a known commit."));
	continue;
	}

	$this->log(pht("Looking for new commits."));

	$refs[] = $this->discoverStreamAncestry(
	new PhabricatorMercurialGraphStream($repository, $commit),
	$commit,
	$close_immediately = true);
	}

	return array_mergev($refs);
	}


	/* -( Internals )---------------------------------------------------------- */


	private function discoverStreamAncestry(
	PhabricatorRepositoryGraphStream $stream,
	$commit,
	$close_immediately) {

	$discover = array($commit);
	$graph = array();
	$seen = array();

	// Find all the reachable, undiscovered commits. Build a graph of the
	// edges.
	while ($discover) {
	$target = array_pop($discover);

	if (empty($graph[$target])) {
	$graph[$target] = array();
	}

	$parents = $stream->getParents($target);
	foreach ($parents as $parent) {
	if ($this->isKnownCommit($parent)) {
	continue;
	}

	$graph[$target][$parent] = true;

	if (empty($seen[$parent])) {
	$seen[$parent] = true;
	$discover[] = $parent;
	}
	}
	}

	// Now, sort them topographically.
	$commits = $this->reduceGraph($graph);

	$refs = array();
	foreach ($commits as $commit) {
	$refs[] = id(new PhabricatorRepositoryCommitRef())
	->setIdentifier($commit)
	->setEpoch($stream->getCommitDate($commit))
	->setCanCloseImmediately($close_immediately);
	}

	return $refs;
	}


	private function reduceGraph(array $edges) {
	foreach ($edges as $commit => $parents) {
	$edges[$commit] = array_keys($parents);
	}

	$graph = new PhutilDirectedScalarGraph();
	$graph->addNodes($edges);

	$commits = $graph->getTopographicallySortedNodes();

	// NOTE: We want the most ancestral nodes first, so we need to reverse the
	// list we get out of AbstractDirectedGraph.
	$commits = array_reverse($commits);

	return $commits;
	}


	private function isKnownCommit($identifier) {
	if (isset($this->commitCache[$identifier])) {
	return true;
	}

	if ($this->repairMode) {
	// In repair mode, rediscover the entire repository, ignoring the
	// database state. We can hit the local cache above, but if we miss it
	// stop the script from going to the database cache.
	return false;
	}

	$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
	'repositoryID = %d AND commitIdentifier = %s',
	$this->getRepository()->getID(),
	$identifier);

	if (!$commit) {
	return false;
	}

	$this->commitCache[$identifier] = true;
	while (count($this->commitCache) > self::MAX_COMMIT_CACHE_SIZE) {
	array_shift($this->commitCache);
	}

	return true;
	}


	/**
	* Sort branches so we process closeable branches first. This makes the
	* whole import process a little cheaper, since we can close these commits
	* the first time through rather than catching them in the refs step.
	*
	* @task internal
	*
	* @param list<DiffusionRepositoryRef> List of branch heads.
	* @return list<DiffusionRepositoryRef> Sorted list of branch heads.
	*/
	private function sortBranches(array $branches) {
	$repository = $this->getRepository();

	$head_branches = array();
	$tail_branches = array();
	foreach ($branches as $branch) {
	$name = $branch->getShortName();

	if ($repository->shouldAutocloseBranch($name)) {
	$head_branches[] = $branch;
	} else {
	$tail_branches[] = $branch;
	}
	}

	return array_merge($head_branches, $tail_branches);
	}

	+
	+ private function recordCommit(
	+ PhabricatorRepository $repository,
	+ $commit_identifier,
	+ $epoch,
	+ $close_immediately) {
	+
	+ $commit = new PhabricatorRepositoryCommit();
	+ $commit->setRepositoryID($repository->getID());
	+ $commit->setCommitIdentifier($commit_identifier);
	+ $commit->setEpoch($epoch);
	+ if ($close_immediately) {
	+ $commit->setImportStatus(PhabricatorRepositoryCommit::IMPORTED_CLOSEABLE);
	+ }
	+
	+ $data = new PhabricatorRepositoryCommitData();
	+
	+ try {
	+ $commit->openTransaction();
	+ $commit->save();
	+ $data->setCommitID($commit->getID());
	+ $data->save();
	+ $commit->saveTransaction();
	+
	+ $this->insertTask($repository, $commit);
	+
	+ queryfx(
	+ $repository->establishConnection('w'),
	+ 'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
	+ VALUES (%d, 1, %d, %d)
	+ ON DUPLICATE KEY UPDATE
	+ size = size + 1,
	+ lastCommitID =
	+ IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID),
	+ epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)',
	+ PhabricatorRepository::TABLE_SUMMARY,
	+ $repository->getID(),
	+ $commit->getID(),
	+ $epoch);
	+
	+ if ($this->repairMode) {
	+ // Normally, the query should throw a duplicate key exception. If we
	+ // reach this in repair mode, we've actually performed a repair.
	+ $this->log(pht('Repaired commit "%s".', $commit_identifier));
	+ }
	+
	+ PhutilEventEngine::dispatchEvent(
	+ new PhabricatorEvent(
	+ PhabricatorEventType::TYPE_DIFFUSION_DIDDISCOVERCOMMIT,
	+ array(
	+ 'repository' => $repository,
	+ 'commit' => $commit,
	+ )));
	+
	+ } catch (AphrontQueryDuplicateKeyException $ex) {
	+ $commit->killTransaction();
	+ // Ignore. This can happen because we discover the same new commit
	+ // more than once when looking at history, or because of races or
	+ // data inconsistency or cosmic radiation; in any case, we're still
	+ // in a good state if we ignore the failure.
	+ }
	+ }
	+
	+ private function insertTask(
	+ PhabricatorRepository $repository,
	+ PhabricatorRepositoryCommit $commit,
	+ $data = array()) {
	+
	+ $vcs = $repository->getVersionControlSystem();
	+ switch ($vcs) {
	+ case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
	+ $class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
	+ break;
	+ case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
	+ $class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
	+ break;
	+ case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
	+ $class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker';
	+ break;
	+ default:
	+ throw new Exception("Unknown repository type '{$vcs}'!");
	+ }
	+
	+ $data['commitID'] = $commit->getID();
	+
	+ PhabricatorWorker::scheduleTask($class, $data);
	+ }
	+
	}
	diff --git a/src/applications/repository/management/PhabricatorRepositoryManagementDiscoverWorkflow.php b/src/applications/repository/management/PhabricatorRepositoryManagementDiscoverWorkflow.php
	index 8a6067dadd..6fac505543 100644
	--- a/src/applications/repository/management/PhabricatorRepositoryManagementDiscoverWorkflow.php
	+++ b/src/applications/repository/management/PhabricatorRepositoryManagementDiscoverWorkflow.php
	@@ -1,52 +1,53 @@
	<?php

	final class PhabricatorRepositoryManagementDiscoverWorkflow
	extends PhabricatorRepositoryManagementWorkflow {

	public function didConstruct() {
	$this
	->setName('discover')
	->setExamples('discover [__options__] __repository__ ...')
	->setSynopsis('Discover __repository__, named by callsign.')
	->setArguments(
	array(
	array(
	'name' => 'verbose',
	'help' => 'Show additional debugging information.',
	),
	array(
	'name' => 'repair',
	'help' => 'Repair a repository with gaps in commit '.
	'history.',
	),
	array(
	'name' => 'repos',
	'wildcard' => true,
	),
	));
	}

	public function execute(PhutilArgumentParser $args) {
	$repos = $this->loadRepositories($args, 'repos');

	if (!$repos) {
	throw new PhutilArgumentUsageException(
	"Specify one or more repositories to discover, by callsign.");
	}

	$console = PhutilConsole::getConsole();
	foreach ($repos as $repo) {
	$console->writeOut("Discovering '%s'...\n", $repo->getCallsign());

	- $daemon = new PhabricatorRepositoryPullLocalDaemon(array());
	- $daemon->setVerbose($args->getArg('verbose'));
	- $daemon->setRepair($args->getArg('repair'));
	- $daemon->discoverRepository($repo);
	+ id(new PhabricatorRepositoryDiscoveryEngine())
	+ ->setRepository($repo)
	+ ->setVerbose($args->getArg('verbose'))
	+ ->setRepairMode($args->getArg('repair'))
	+ ->discoverCommits();
	}

	$console->writeOut("Done.\n");

	return 0;
	}

	}

File Metadata

Mime Type: text/x-diff
Expires: Tue, Dec 2, 4:43 AM (21 h, 18 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 431635
Default Alt Text: (33 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions