Page MenuHomestyx hydra

No OneTemporary

diff --git a/src/applications/diffusion/conduit/ConduitAPI_diffusion_commitbranchesquery_Method.php b/src/applications/diffusion/conduit/ConduitAPI_diffusion_commitbranchesquery_Method.php
index a8e9000787..65d503d114 100644
--- a/src/applications/diffusion/conduit/ConduitAPI_diffusion_commitbranchesquery_Method.php
+++ b/src/applications/diffusion/conduit/ConduitAPI_diffusion_commitbranchesquery_Method.php
@@ -1,61 +1,63 @@
<?php
/**
* @group conduit
*/
final class ConduitAPI_diffusion_commitbranchesquery_Method
extends ConduitAPI_diffusion_abstractquery_Method {
public function getMethodDescription() {
return 'Determine what branches contain a commit in a repository.';
}
public function defineReturnType() {
return 'array';
}
protected function defineCustomParamTypes() {
return array(
'commit' => 'required string',
);
}
protected function getGitResult(ConduitAPIRequest $request) {
$drequest = $this->getDiffusionRequest();
$repository = $drequest->getRepository();
$commit = $request->getValue('commit');
// NOTE: We can't use DiffusionLowLevelGitRefQuery here because
// `git for-each-ref` does not support `--contains`.
if ($repository->isWorkingCopyBare()) {
list($contains) = $repository->execxLocalCommand(
'branch --verbose --no-abbrev --contains %s',
$commit);
return DiffusionGitBranch::parseLocalBranchOutput(
$contains);
} else {
list($contains) = $repository->execxLocalCommand(
'branch -r --verbose --no-abbrev --contains %s',
$commit);
return DiffusionGitBranch::parseRemoteBranchOutput(
$contains,
DiffusionBranchInformation::DEFAULT_GIT_REMOTE);
}
}
protected function getMercurialResult(ConduitAPIRequest $request) {
$drequest = $this->getDiffusionRequest();
$repository = $drequest->getRepository();
$commit = $request->getValue('commit');
+ // TODO: This should use `branches`.
+
list($contains) = $repository->execxLocalCommand(
'log --template %s --limit 1 --rev %s --',
'{branch}',
$commit);
return array(
trim($contains) => $commit,
);
}
}
diff --git a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php
index 7ce6297d57..4f1b80534c 100644
--- a/src/applications/diffusion/engine/DiffusionCommitHookEngine.php
+++ b/src/applications/diffusion/engine/DiffusionCommitHookEngine.php
@@ -1,460 +1,668 @@
<?php
/**
* @task config Configuring the Hook Engine
* @task hook Hook Execution
* @task git Git Hooks
* @task hg Mercurial Hooks
* @task svn Subversion Hooks
* @task internal Internals
*/
final class DiffusionCommitHookEngine extends Phobject {
const ENV_USER = 'PHABRICATOR_USER';
const ENV_REMOTE_ADDRESS = 'PHABRICATOR_REMOTE_ADDRESS';
const ENV_REMOTE_PROTOCOL = 'PHABRICATOR_REMOTE_PROTOCOL';
const EMPTY_HASH = '0000000000000000000000000000000000000000';
private $viewer;
private $repository;
private $stdin;
private $subversionTransaction;
private $subversionRepository;
private $remoteAddress;
private $remoteProtocol;
private $transactionKey;
/* -( Config )------------------------------------------------------------- */
public function setRemoteProtocol($remote_protocol) {
$this->remoteProtocol = $remote_protocol;
return $this;
}
public function getRemoteProtocol() {
return $this->remoteProtocol;
}
public function setRemoteAddress($remote_address) {
$this->remoteAddress = $remote_address;
return $this;
}
public function getRemoteAddress() {
return $this->remoteAddress;
}
private function getRemoteAddressForLog() {
// If whatever we have here isn't a valid IPv4 address, just store `null`.
// Older versions of PHP return `-1` on failure instead of `false`.
$remote_address = $this->getRemoteAddress();
$remote_address = max(0, ip2long($remote_address));
$remote_address = nonempty($remote_address, null);
return $remote_address;
}
private function getTransactionKey() {
if (!$this->transactionKey) {
$entropy = Filesystem::readRandomBytes(64);
$this->transactionKey = PhabricatorHash::digestForIndex($entropy);
}
return $this->transactionKey;
}
public function setSubversionTransactionInfo($transaction, $repository) {
$this->subversionTransaction = $transaction;
$this->subversionRepository = $repository;
return $this;
}
public function setStdin($stdin) {
$this->stdin = $stdin;
return $this;
}
public function getStdin() {
return $this->stdin;
}
public function setRepository(PhabricatorRepository $repository) {
$this->repository = $repository;
return $this;
}
public function getRepository() {
return $this->repository;
}
public function setViewer(PhabricatorUser $viewer) {
$this->viewer = $viewer;
return $this;
}
public function getViewer() {
return $this->viewer;
}
/* -( Hook Execution )----------------------------------------------------- */
public function execute() {
$ref_updates = $this->findRefUpdates();
$all_updates = $ref_updates;
$caught = null;
try {
try {
$this->rejectDangerousChanges($ref_updates);
} catch (DiffusionCommitHookRejectException $ex) {
// If we're rejecting dangerous changes, flag everything that we've
// seen as rejected so it's clear that none of it was accepted.
foreach ($all_updates as $update) {
$update->setRejectCode(
PhabricatorRepositoryPushLog::REJECT_DANGEROUS);
}
throw $ex;
}
// TODO: Fire ref herald rules.
$content_updates = $this->findContentUpdates($ref_updates);
$all_updates = array_merge($all_updates, $content_updates);
// TODO: Fire content Herald rules.
// TODO: Fire external hooks.
// If we make it this far, we're accepting these changes. Mark all the
// logs as accepted.
foreach ($all_updates as $update) {
$update->setRejectCode(PhabricatorRepositoryPushLog::REJECT_ACCEPT);
}
} catch (Exception $ex) {
// We'll throw this again in a minute, but we want to save all the logs
// first.
$caught = $ex;
}
// Save all the logs no matter what the outcome was.
foreach ($all_updates as $update) {
$update->save();
}
if ($caught) {
throw $caught;
}
return 0;
}
private function findRefUpdates() {
$type = $this->getRepository()->getVersionControlSystem();
switch ($type) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
return $this->findGitRefUpdates();
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
return $this->findMercurialRefUpdates();
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
return $this->findSubversionRefUpdates();
default:
throw new Exception(pht('Unsupported repository type "%s"!', $type));
}
}
private function rejectDangerousChanges(array $ref_updates) {
assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog');
$repository = $this->getRepository();
if ($repository->shouldAllowDangerousChanges()) {
return;
}
$flag_dangerous = PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS;
foreach ($ref_updates as $ref_update) {
if (!$ref_update->hasChangeFlags($flag_dangerous)) {
// This is not a dangerous change.
continue;
}
// We either have a branch deletion or a non fast-forward branch update.
// Format a message and reject the push.
$message = pht(
"DANGEROUS CHANGE: %s\n".
"Dangerous change protection is enabled for this repository.\n".
"Edit the repository configuration before making dangerous changes.",
$ref_update->getDangerousChangeDescription());
throw new DiffusionCommitHookRejectException($message);
}
}
private function findContentUpdates(array $ref_updates) {
assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog');
$type = $this->getRepository()->getVersionControlSystem();
switch ($type) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
return $this->findGitContentUpdates($ref_updates);
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
return $this->findMercurialContentUpdates($ref_updates);
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
return $this->findSubversionContentUpdates($ref_updates);
default:
throw new Exception(pht('Unsupported repository type "%s"!', $type));
}
}
/* -( Git )---------------------------------------------------------------- */
private function findGitRefUpdates() {
$ref_updates = array();
// First, parse stdin, which lists all the ref changes. The input looks
// like this:
//
// <old hash> <new hash> <ref>
$stdin = $this->getStdin();
$lines = phutil_split_lines($stdin, $retain_endings = false);
foreach ($lines as $line) {
$parts = explode(' ', $line, 3);
if (count($parts) != 3) {
throw new Exception(pht('Expected "old new ref", got "%s".', $line));
}
$ref_old = $parts[0];
$ref_new = $parts[1];
$ref_raw = $parts[2];
if (preg_match('(^refs/heads/)', $ref_raw)) {
$ref_type = PhabricatorRepositoryPushLog::REFTYPE_BRANCH;
} else if (preg_match('(^refs/tags/)', $ref_raw)) {
$ref_type = PhabricatorRepositoryPushLog::REFTYPE_TAG;
} else {
$ref_type = PhabricatorRepositoryPushLog::REFTYPE_UNKNOWN;
}
$ref_update = $this->newPushLog()
->setRefType($ref_type)
->setRefName($ref_raw)
->setRefOld($ref_old)
->setRefNew($ref_new);
$ref_updates[] = $ref_update;
}
$this->findGitMergeBases($ref_updates);
$this->findGitChangeFlags($ref_updates);
return $ref_updates;
}
private function findGitMergeBases(array $ref_updates) {
assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog');
$futures = array();
foreach ($ref_updates as $key => $ref_update) {
// If the old hash is "00000...", the ref is being created (either a new
// branch, or a new tag). If the new hash is "00000...", the ref is being
// deleted. If both are nonempty, the ref is being updated. For updates,
// we'll figure out the `merge-base` of the old and new objects here. This
// lets us reject non-FF changes cheaply; later, we'll figure out exactly
// which commits are new.
$ref_old = $ref_update->getRefOld();
$ref_new = $ref_update->getRefNew();
if (($ref_old === self::EMPTY_HASH) ||
($ref_new === self::EMPTY_HASH)) {
continue;
}
$futures[$key] = $this->getRepository()->getLocalCommandFuture(
'merge-base %s %s',
$ref_old,
$ref_new);
}
foreach (Futures($futures)->limit(8) as $key => $future) {
// If 'old' and 'new' have no common ancestors (for example, a force push
// which completely rewrites a ref), `git merge-base` will exit with
// an error and no output. It would be nice to find a positive test
// for this instead, but I couldn't immediately come up with one. See
// T4224. Assume this means there are no ancestors.
list($err, $stdout) = $future->resolve();
if ($err) {
$merge_base = null;
} else {
$merge_base = rtrim($stdout, "\n");
}
$ref_update->setMergeBase($merge_base);
}
return $ref_updates;
}
private function findGitChangeFlags(array $ref_updates) {
assert_instances_of($ref_updates, 'PhabricatorRepositoryPushLog');
foreach ($ref_updates as $key => $ref_update) {
$ref_old = $ref_update->getRefOld();
$ref_new = $ref_update->getRefNew();
$ref_type = $ref_update->getRefType();
$ref_flags = 0;
$dangerous = null;
if ($ref_old === self::EMPTY_HASH) {
$ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD;
} else if ($ref_new === self::EMPTY_HASH) {
$ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE;
if ($ref_type == PhabricatorRepositoryPushLog::REFTYPE_BRANCH) {
$ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS;
$dangerous = pht(
"The change you're attempting to push deletes the branch '%s'.",
$ref_update->getRefName());
}
} else {
$merge_base = $ref_update->getMergeBase();
if ($merge_base == $ref_old) {
// This is a fast-forward update to an existing branch.
// These are safe.
$ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND;
} else {
$ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_REWRITE;
// For now, we don't consider deleting or moving tags to be a
// "dangerous" update. It's way harder to get wrong and should be easy
// to recover from once we have better logging. Only add the dangerous
// flag if this ref is a branch.
if ($ref_type == PhabricatorRepositoryPushLog::REFTYPE_BRANCH) {
$ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS;
$dangerous = pht(
- "DANGEROUS CHANGE: The change you're attempting to push updates ".
- "the branch '%s' from '%s' to '%s', but this is not a ".
- "fast-forward. Pushes which rewrite published branch history ".
- "are dangerous.",
+ "The change you're attempting to push updates the branch '%s' ".
+ "from '%s' to '%s', but this is not a fast-forward. Pushes ".
+ "which rewrite published branch history are dangerous.",
$ref_update->getRefName(),
$ref_update->getRefOldShort(),
$ref_update->getRefNewShort());
}
}
}
$ref_update->setChangeFlags($ref_flags);
if ($dangerous !== null) {
$ref_update->attachDangerousChangeDescription($dangerous);
}
}
return $ref_updates;
}
private function findGitContentUpdates(array $ref_updates) {
$flag_delete = PhabricatorRepositoryPushLog::CHANGEFLAG_DELETE;
$futures = array();
foreach ($ref_updates as $key => $ref_update) {
if ($ref_update->hasChangeFlags($flag_delete)) {
// Deleting a branch or tag can never create any new commits.
continue;
}
// NOTE: This piece of magic finds all new commits, by walking backward
// from the new value to the value of *any* existing ref in the
// repository. Particularly, this will cover the cases of a new branch, a
// completely moved tag, etc.
$futures[$key] = $this->getRepository()->getLocalCommandFuture(
'log --format=%s %s --not --all',
'%H',
$ref_update->getRefNew());
}
$content_updates = array();
foreach (Futures($futures)->limit(8) as $key => $future) {
list($stdout) = $future->resolvex();
if (!strlen(trim($stdout))) {
// This change doesn't have any new commits. One common case of this
// is creating a new tag which points at an existing commit.
continue;
}
$commits = phutil_split_lines($stdout, $retain_newlines = false);
foreach ($commits as $commit) {
$content_updates[$commit] = $this->newPushLog()
->setRefType(PhabricatorRepositoryPushLog::REFTYPE_COMMIT)
->setRefNew($commit)
->setChangeFlags(PhabricatorRepositoryPushLog::CHANGEFLAG_ADD);
}
}
return $content_updates;
}
/* -( Mercurial )---------------------------------------------------------- */
private function findMercurialRefUpdates() {
- // TODO: Implement.
- return array();
+ $hg_node = getenv('HG_NODE');
+ if (!$hg_node) {
+ throw new Exception(pht('Expected HG_NODE in environment!'));
+ }
+
+ // NOTE: We need to make sure this is passed to subprocesses, or they won't
+ // be able to see new commits. Mercurial uses this as a marker to determine
+ // whether the pending changes are visible or not.
+ $_ENV['HG_PENDING'] = getenv('HG_PENDING');
+ $repository = $this->getRepository();
+
+ $futures = array();
+
+ foreach (array('old', 'new') as $key) {
+ $futures[$key] = $repository->getLocalCommandFuture(
+ 'heads --template %s',
+ '{node}\1{branches}\2');
+ }
+ // Wipe HG_PENDING out of the old environment so we see the pre-commit
+ // state of the repository.
+ $futures['old']->updateEnv('HG_PENDING', null);
+
+ $futures['commits'] = $repository->getLocalCommandFuture(
+ "log --rev %s --rev tip --template %s",
+ hgsprintf('%s', $hg_node),
+ '{node}\1{branches}\2');
+
+ // Resolve all of the futures now. We don't need the 'commits' future yet,
+ // but it simplifies the logic to just get it out of the way.
+ foreach (Futures($futures) as $future) {
+ $future->resolvex();
+ }
+
+ list($commit_raw) = $futures['commits']->resolvex();
+ $commit_map = $this->parseMercurialCommits($commit_raw);
+
+ list($old_raw) = $futures['old']->resolvex();
+ $old_refs = $this->parseMercurialHeads($old_raw);
+
+ list($new_raw) = $futures['new']->resolvex();
+ $new_refs = $this->parseMercurialHeads($new_raw);
+
+ $all_refs = array_keys($old_refs + $new_refs);
+
+ $ref_updates = array();
+ foreach ($all_refs as $ref) {
+ $old_heads = idx($old_refs, $ref, array());
+ $new_heads = idx($new_refs, $ref, array());
+
+ sort($old_heads);
+ sort($new_heads);
+
+ if ($old_heads === $new_heads) {
+ // No changes to this branch, so skip it.
+ continue;
+ }
+
+ if (!$new_heads) {
+ if ($old_heads) {
+ // It looks like this push deletes a branch, but that isn't possible
+ // in Mercurial, so something is going wrong here. Bail out.
+ throw new Exception(
+ pht(
+ 'Mercurial repository has no new head for branch "%s" after '.
+ 'push. This is unexpected; rejecting change.'));
+ } else {
+ // Obviously, this should never be possible either, as it makes
+ // no sense. Explode.
+ throw new Exception(
+ pht(
+ 'Mercurial repository has no new or old heads for branch "%s" '.
+ 'after push. This makes no sense; rejecting change.'));
+ }
+ }
+
+ $stray_heads = array();
+ if (count($old_heads) > 1) {
+ // HORRIBLE: In Mercurial, branches can have multiple heads. If the
+ // old branch had multiple heads, we need to figure out which new
+ // heads descend from which old heads, so we can tell whether you're
+ // actively creating new heads (dangerous) or just working in a
+ // repository that's already full of garbage (strongly discouraged but
+ // not as inherently dangerous). These cases should be very uncommon.
+
+ $dfutures = array();
+ foreach ($old_heads as $old_head) {
+ $dfutures[$old_head] = $repository->getLocalCommandFuture(
+ 'log --rev %s --template %s',
+ hgsprintf('(descendants(%s) and head())', $old_head),
+ '{node}\1');
+ }
+
+ $head_map = array();
+ foreach (Futures($dfutures) as $future_head => $dfuture) {
+ list($stdout) = $dfuture->resolvex();
+ $head_map[$future_head] = array_filter(explode("\1", $stdout));
+ }
+
+ // Now, find all the new stray heads this push creates, if any. These
+ // are new heads which do not descend from the old heads.
+ $seen = array_fuse(array_mergev($head_map));
+ foreach ($new_heads as $new_head) {
+ if (empty($seen[$new_head])) {
+ $head_map[self::EMPTY_HASH][] = $new_head;
+ }
+ }
+ } else if ($old_heads) {
+ $head_map[head($old_heads)] = $new_heads;
+ } else {
+ $head_map[self::EMPTY_HASH] = $new_heads;
+ }
+
+ foreach ($head_map as $old_head => $child_heads) {
+ foreach ($child_heads as $new_head) {
+ if ($new_head === $old_head) {
+ continue;
+ }
+
+ $ref_flags = 0;
+ $dangerous = null;
+ if ($old_head == self::EMPTY_HASH) {
+ $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_ADD;
+ } else {
+ $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_APPEND;
+ }
+
+ $splits_existing_head = (count($child_heads) > 1);
+ $creates_duplicate_head = ($old_head == self::EMPTY_HASH) &&
+ (count($head_map) > 1);
+
+ if ($splits_existing_head || $creates_duplicate_head) {
+ $readable_child_heads = array();
+ foreach ($child_heads as $child_head) {
+ $readable_child_heads[] = substr($child_head, 0, 12);
+ }
+
+ $ref_flags |= PhabricatorRepositoryPushLog::CHANGEFLAG_DANGEROUS;
+
+ if ($splits_existing_head) {
+ // We're splitting an existing head into two or more heads.
+ // This is dangerous, and a super bad idea. Note that we're only
+ // raising this if you're actively splitting a branch head. If a
+ // head split in the past, we don't consider appends to it
+ // to be dangerous.
+ $dangerous = pht(
+ "The change you're attempting to push splits the head of ".
+ "branch '%s' into multiple heads: %s. This is inadvisable ".
+ "and dangerous.",
+ $ref,
+ implode(', ', $readable_child_heads));
+ } else {
+ // We're adding a second (or more) head to a branch. The new
+ // head is not a descendant of any old head.
+ $dangerous = pht(
+ "The change you're attempting to push creates new, divergent ".
+ "heads for the branch '%s': %s. This is inadvisable and ".
+ "dangerous.",
+ $ref,
+ implode(', ', $readable_child_heads));
+ }
+ }
+
+ $ref_update = $this->newPushLog()
+ ->setRefType(PhabricatorRepositoryPushLog::REFTYPE_BRANCH)
+ ->setRefName($ref)
+ ->setRefOld($old_head)
+ ->setRefNew($new_head)
+ ->setChangeFlags($ref_flags);
+
+ if ($dangerous !== null) {
+ $ref_update->attachDangerousChangeDescription($dangerous);
+ }
+
+ $ref_updates[] = $ref_update;
+ }
+ }
+ }
+
+ return $ref_updates;
+ }
+
+ private function parseMercurialCommits($raw) {
+ $commits_lines = explode("\2", $raw);
+ $commits_lines = array_filter($commits_lines);
+ $commit_map = array();
+ foreach ($commits_lines as $commit_line) {
+ list($node, $branches_raw) = explode("\1", $commit_line);
+
+ if (!strlen($branches_raw)) {
+ $branches = array('default');
+ } else {
+ $branches = explode(' ', $branches_raw);
+ }
+
+ $commit_map[$node] = $branches;
+ }
+
+ return $commit_map;
+ }
+
+ private function parseMercurialHeads($raw) {
+ $heads_map = $this->parseMercurialCommits($raw);
+
+ $heads = array();
+ foreach ($heads_map as $commit => $branches) {
+ foreach ($branches as $branch) {
+ $heads[$branch][] = $commit;
+ }
+ }
+
+ return $heads;
}
private function findMercurialContentUpdates(array $ref_updates) {
// TODO: Implement.
return array();
}
/* -( Subversion )--------------------------------------------------------- */
private function findSubversionRefUpdates() {
// TODO: Implement.
return array();
}
private function findSubversionContentUpdates(array $ref_updates) {
// TODO: Implement.
return array();
}
/* -( Internals )---------------------------------------------------------- */
private function newPushLog() {
// NOTE: By default, we create these with REJECT_BROKEN as the reject
// code. This indicates a broken hook, and covers the case where we
// encounter some unexpected exception and consequently reject the changes.
return PhabricatorRepositoryPushLog::initializeNewLog($this->getViewer())
->attachRepository($this->getRepository())
->setRepositoryPHID($this->getRepository()->getPHID())
->setEpoch(time())
->setRemoteAddress($this->getRemoteAddressForLog())
->setRemoteProtocol($this->getRemoteProtocol())
->setTransactionKey($this->getTransactionKey())
->setRejectCode(PhabricatorRepositoryPushLog::REJECT_BROKEN)
->setRejectDetails(null);
}
}
diff --git a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
index 0bf9007d03..62cb4eeb3f 100644
--- a/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
+++ b/src/applications/repository/daemon/PhabricatorRepositoryPullLocalDaemon.php
@@ -1,853 +1,853 @@
<?php
/**
* Run pull commands on local working copies to keep them up to date. This
* daemon handles all repository types.
*
* By default, the daemon pulls **every** repository. If you want it to be
* responsible for only some repositories, you can launch it with a list of
* PHIDs or callsigns:
*
* ./phd launch repositorypulllocal -- X Q Z
*
* You can also launch a daemon which is responsible for all //but// one or
* more repositories:
*
* ./phd launch repositorypulllocal -- --not A --not B
*
* If you have a very large number of repositories and some aren't being pulled
* as frequently as you'd like, you can either change the pull frequency of
* the less-important repositories to a larger number (so the daemon will skip
* them more often) or launch one daemon for all the less-important repositories
* and one for the more important repositories (or one for each more important
* repository).
*
* @task pull Pulling Repositories
* @task git Git Implementation
* @task hg Mercurial Implementation
*/
final class PhabricatorRepositoryPullLocalDaemon
extends PhabricatorDaemon {
private $commitCache = array();
private $repair;
private $discoveryEngines = array();
public function setRepair($repair) {
$this->repair = $repair;
return $this;
}
/* -( Pulling Repositories )----------------------------------------------- */
/**
* @task pull
*/
public function run() {
$argv = $this->getArgv();
array_unshift($argv, __CLASS__);
$args = new PhutilArgumentParser($argv);
$args->parse(
array(
array(
'name' => 'no-discovery',
'help' => 'Pull only, without discovering commits.',
),
array(
'name' => 'not',
'param' => 'repository',
'repeat' => true,
'help' => 'Do not pull __repository__.',
),
array(
'name' => 'repositories',
'wildcard' => true,
'help' => 'Pull specific __repositories__ instead of all.',
),
));
$no_discovery = $args->getArg('no-discovery');
$repo_names = $args->getArg('repositories');
$exclude_names = $args->getArg('not');
// Each repository has an individual pull frequency; after we pull it,
// wait that long to pull it again. When we start up, try to pull everything
// serially.
$retry_after = array();
$min_sleep = 15;
while (true) {
$repositories = $this->loadRepositories($repo_names);
if ($exclude_names) {
$exclude = $this->loadRepositories($exclude_names);
$repositories = array_diff_key($repositories, $exclude);
}
// Shuffle the repositories, then re-key the array since shuffle()
// discards keys. This is mostly for startup, we'll use soft priorities
// later.
shuffle($repositories);
$repositories = mpull($repositories, null, 'getID');
// If any repositories have the NEEDS_UPDATE flag set, pull them
// as soon as possible.
$type_need_update = PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE;
$need_update_messages = id(new PhabricatorRepositoryStatusMessage())
->loadAllWhere('statusType = %s', $type_need_update);
foreach ($need_update_messages as $message) {
$retry_after[$message->getRepositoryID()] = time();
}
// If any repositories were deleted, remove them from the retry timer map
// so we don't end up with a retry timer that never gets updated and
// causes us to sleep for the minimum amount of time.
$retry_after = array_select_keys(
$retry_after,
array_keys($repositories));
// Assign soft priorities to repositories based on how frequently they
// should pull again.
asort($retry_after);
$repositories = array_select_keys(
$repositories,
array_keys($retry_after)) + $repositories;
foreach ($repositories as $id => $repository) {
$after = idx($retry_after, $id, 0);
if ($after > time()) {
continue;
}
$tracked = $repository->isTracked();
if (!$tracked) {
continue;
}
$callsign = $repository->getCallsign();
try {
$this->log("Updating repository '{$callsign}'.");
id(new PhabricatorRepositoryPullEngine())
->setRepository($repository)
->pullRepository();
if (!$no_discovery) {
// TODO: It would be nice to discover only if we pulled something,
// but this isn't totally trivial. It's slightly more complicated
// with hosted repositories, too.
$lock_name = get_class($this).':'.$callsign;
$lock = PhabricatorGlobalLock::newLock($lock_name);
$lock->lock();
$repository->writeStatusMessage(
PhabricatorRepositoryStatusMessage::TYPE_NEEDS_UPDATE,
null);
try {
$this->discoverRepository($repository);
$repository->writeStatusMessage(
PhabricatorRepositoryStatusMessage::TYPE_FETCH,
PhabricatorRepositoryStatusMessage::CODE_OKAY);
} catch (Exception $ex) {
$repository->writeStatusMessage(
PhabricatorRepositoryStatusMessage::TYPE_FETCH,
PhabricatorRepositoryStatusMessage::CODE_ERROR,
array(
'message' => pht(
'Error updating working copy: %s', $ex->getMessage()),
));
$lock->unlock();
throw $ex;
}
$lock->unlock();
}
$sleep_for = $repository->getDetail('pull-frequency', $min_sleep);
$retry_after[$id] = time() + $sleep_for;
} catch (PhutilLockException $ex) {
$retry_after[$id] = time() + $min_sleep;
$this->log("Failed to acquire lock.");
} catch (Exception $ex) {
$retry_after[$id] = time() + $min_sleep;
$proxy = new PhutilProxyException(
"Error while fetching changes to the '{$callsign}' repository.",
$ex);
phlog($proxy);
}
$this->stillWorking();
}
if ($retry_after) {
$sleep_until = max(min($retry_after), time() + $min_sleep);
} else {
$sleep_until = time() + $min_sleep;
}
$this->sleep($sleep_until - time());
}
}
/**
* @task pull
*/
protected function loadRepositories(array $names) {
$query = id(new PhabricatorRepositoryQuery())
->setViewer($this->getViewer());
if ($names) {
$query->withCallsigns($names);
}
$repos = $query->execute();
if ($names) {
$by_callsign = mpull($repos, null, 'getCallsign');
foreach ($names as $name) {
if (empty($by_callsign[$name])) {
throw new Exception(
"No repository exists with callsign '{$name}'!");
}
}
}
return $repos;
}
public function discoverRepository(PhabricatorRepository $repository) {
$vcs = $repository->getVersionControlSystem();
$result = null;
$refs = null;
switch ($vcs) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
$result = $this->executeGitDiscover($repository);
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
$refs = $this->getDiscoveryEngine($repository)
->discoverCommits();
break;
default:
throw new Exception("Unknown VCS '{$vcs}'!");
}
if ($refs !== null) {
foreach ($refs as $ref) {
$this->recordCommit(
$repository,
$ref->getIdentifier(),
$ref->getEpoch(),
$ref->getBranch());
}
}
$this->checkIfRepositoryIsFullyImported($repository);
try {
$this->pushToMirrors($repository);
} catch (Exception $ex) {
// TODO: We should report these into the UI properly, but for
// now just complain. These errors are much less severe than
// pull errors.
phlog($ex);
}
if ($refs !== null) {
return (bool)count($refs);
} else {
return $result;
}
}
private function getDiscoveryEngine(PhabricatorRepository $repository) {
$id = $repository->getID();
if (empty($this->discoveryEngines[$id])) {
$engine = id(new PhabricatorRepositoryDiscoveryEngine())
->setRepository($repository)
->setVerbose($this->getVerbose())
->setRepairMode($this->repair);
$this->discoveryEngines[$id] = $engine;
}
return $this->discoveryEngines[$id];
}
private function isKnownCommit(
PhabricatorRepository $repository,
$target) {
if ($this->getCache($repository, $target)) {
return true;
}
if ($this->repair) {
// In repair mode, rediscover the entire repository, ignoring the
// database state. We can hit the local cache above, but if we miss it
// stop the script from going to the database cache.
return false;
}
$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
'repositoryID = %d AND commitIdentifier = %s',
$repository->getID(),
$target);
if (!$commit) {
return false;
}
$this->setCache($repository, $target);
while (count($this->commitCache) > 2048) {
array_shift($this->commitCache);
}
return true;
}
private function isKnownCommitOnAnyAutocloseBranch(
PhabricatorRepository $repository,
$target) {
$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
'repositoryID = %d AND commitIdentifier = %s',
$repository->getID(),
$target);
if (!$commit) {
$callsign = $repository->getCallsign();
$console = PhutilConsole::getConsole();
$console->writeErr(
"WARNING: Repository '%s' is missing commits ('%s' is missing from ".
"history). Run '%s' to repair the repository.\n",
$callsign,
$target,
"bin/repository discover --repair {$callsign}");
return false;
}
$data = $commit->loadCommitData();
if (!$data) {
return false;
}
if ($repository->shouldAutocloseCommit($commit, $data)) {
return true;
}
return false;
}
private function recordCommit(
PhabricatorRepository $repository,
$commit_identifier,
$epoch,
$branch = null) {
$commit = new PhabricatorRepositoryCommit();
$commit->setRepositoryID($repository->getID());
$commit->setCommitIdentifier($commit_identifier);
$commit->setEpoch($epoch);
$data = new PhabricatorRepositoryCommitData();
if ($branch) {
$data->setCommitDetail('seenOnBranches', array($branch));
}
try {
$commit->openTransaction();
$commit->save();
$data->setCommitID($commit->getID());
$data->save();
$commit->saveTransaction();
$this->insertTask($repository, $commit);
queryfx(
$repository->establishConnection('w'),
'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
VALUES (%d, 1, %d, %d)
ON DUPLICATE KEY UPDATE
size = size + 1,
lastCommitID =
IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID),
epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)',
PhabricatorRepository::TABLE_SUMMARY,
$repository->getID(),
$commit->getID(),
$epoch);
if ($this->repair) {
// Normally, the query should throw a duplicate key exception. If we
// reach this in repair mode, we've actually performed a repair.
$this->log("Repaired commit '{$commit_identifier}'.");
}
$this->setCache($repository, $commit_identifier);
PhutilEventEngine::dispatchEvent(
new PhabricatorEvent(
PhabricatorEventType::TYPE_DIFFUSION_DIDDISCOVERCOMMIT,
array(
'repository' => $repository,
'commit' => $commit,
)));
} catch (AphrontQueryDuplicateKeyException $ex) {
$commit->killTransaction();
// Ignore. This can happen because we discover the same new commit
// more than once when looking at history, or because of races or
// data inconsistency or cosmic radiation; in any case, we're still
// in a good state if we ignore the failure.
$this->setCache($repository, $commit_identifier);
}
}
private function updateCommit(
PhabricatorRepository $repository,
$commit_identifier,
$branch) {
$commit = id(new PhabricatorRepositoryCommit())->loadOneWhere(
'repositoryID = %d AND commitIdentifier = %s',
$repository->getID(),
$commit_identifier);
if (!$commit) {
// This can happen if the phabricator DB doesn't have the commit info,
// or the commit is so big that phabricator couldn't parse it. In this
// case we just ignore it.
return;
}
$data = id(new PhabricatorRepositoryCommitData())->loadOneWhere(
'commitID = %d',
$commit->getID());
if (!$data) {
$data = new PhabricatorRepositoryCommitData();
$data->setCommitID($commit->getID());
}
$branches = $data->getCommitDetail('seenOnBranches', array());
$branches[] = $branch;
$data->setCommitDetail('seenOnBranches', $branches);
$data->save();
$this->insertTask(
$repository,
$commit,
array(
'only' => true
));
}
private function insertTask(
PhabricatorRepository $repository,
PhabricatorRepositoryCommit $commit,
$data = array()) {
$vcs = $repository->getVersionControlSystem();
switch ($vcs) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
$class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
$class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
$class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker';
break;
default:
throw new Exception("Unknown repository type '{$vcs}'!");
}
$data['commitID'] = $commit->getID();
PhabricatorWorker::scheduleTask($class, $data);
}
private function setCache(
PhabricatorRepository $repository,
$commit_identifier) {
$key = $this->getCacheKey($repository, $commit_identifier);
$this->commitCache[$key] = true;
}
private function getCache(
PhabricatorRepository $repository,
$commit_identifier) {
$key = $this->getCacheKey($repository, $commit_identifier);
return idx($this->commitCache, $key, false);
}
private function getCacheKey(
PhabricatorRepository $repository,
$commit_identifier) {
return $repository->getID().':'.$commit_identifier;
}
private function checkIfRepositoryIsFullyImported(
PhabricatorRepository $repository) {
// Check if the repository has the "Importing" flag set. We want to clear
// the flag if we can.
$importing = $repository->getDetail('importing');
if (!$importing) {
// This repository isn't marked as "Importing", so we're done.
return;
}
// Look for any commit which hasn't imported.
$unparsed_commit = queryfx_one(
$repository->establishConnection('r'),
'SELECT * FROM %T WHERE repositoryID = %d AND importStatus != %d
LIMIT 1',
id(new PhabricatorRepositoryCommit())->getTableName(),
$repository->getID(),
PhabricatorRepositoryCommit::IMPORTED_ALL);
if ($unparsed_commit) {
// We found a commit which still needs to import, so we can't clear the
// flag.
return;
}
// Clear the "importing" flag.
$repository->openTransaction();
$repository->beginReadLocking();
$repository = $repository->reload();
$repository->setDetail('importing', false);
$repository->save();
$repository->endReadLocking();
$repository->saveTransaction();
}
/* -( Git Implementation )------------------------------------------------- */
/**
* @task git
*/
private function executeGitDiscover(
PhabricatorRepository $repository) {
if (!$repository->isHosted()) {
$this->verifyOrigin($repository);
}
$refs = id(new DiffusionLowLevelGitRefQuery())
->setRepository($repository)
->withIsOriginBranch(true)
->execute();
$branches = mpull($refs, 'getCommitIdentifier', 'getShortName');
if (!$branches) {
// This repository has no branches at all, so we don't need to do
// anything. Generally, this means the repository is empty.
return;
}
$callsign = $repository->getCallsign();
$tracked_something = false;
$this->log("Discovering commits in repository '{$callsign}'...");
foreach ($branches as $name => $commit) {
$this->log("Examining branch '{$name}', at {$commit}.");
if (!$repository->shouldTrackBranch($name)) {
$this->log("Skipping, branch is untracked.");
continue;
}
$tracked_something = true;
if ($this->isKnownCommit($repository, $commit)) {
$this->log("Skipping, HEAD is known.");
continue;
}
$this->log("Looking for new commits.");
$this->executeGitDiscoverCommit($repository, $commit, $name, false);
}
if (!$tracked_something) {
$repo_name = $repository->getName();
$repo_callsign = $repository->getCallsign();
throw new Exception(
"Repository r{$repo_callsign} '{$repo_name}' has no tracked branches! ".
"Verify that your branch filtering settings are correct.");
}
$this->log("Discovering commits on autoclose branches...");
foreach ($branches as $name => $commit) {
$this->log("Examining branch '{$name}', at {$commit}'.");
if (!$repository->shouldTrackBranch($name)) {
$this->log("Skipping, branch is untracked.");
continue;
}
if (!$repository->shouldAutocloseBranch($name)) {
$this->log("Skipping, branch is not autoclose.");
continue;
}
if ($this->isKnownCommitOnAnyAutocloseBranch($repository, $commit)) {
$this->log("Skipping, commit is known on an autoclose branch.");
continue;
}
$this->log("Looking for new autoclose commits.");
$this->executeGitDiscoverCommit($repository, $commit, $name, true);
}
}
/**
* @task git
*/
private function executeGitDiscoverCommit(
PhabricatorRepository $repository,
$commit,
$branch,
$autoclose) {
$discover = array($commit);
$insert = array($commit);
$seen_parent = array();
$stream = new PhabricatorGitGraphStream($repository, $commit);
while (true) {
$target = array_pop($discover);
$parents = $stream->getParents($target);
foreach ($parents as $parent) {
if (isset($seen_parent[$parent])) {
// We end up in a loop here somehow when we parse Arcanist if we
// don't do this. TODO: Figure out why and draw a pretty diagram
// since it's not evident how parsing a DAG with this causes the
// loop to stop terminating.
continue;
}
$seen_parent[$parent] = true;
if ($autoclose) {
$known = $this->isKnownCommitOnAnyAutocloseBranch(
$repository,
$parent);
} else {
$known = $this->isKnownCommit($repository, $parent);
}
if (!$known) {
$this->log("Discovered commit '{$parent}'.");
$discover[] = $parent;
$insert[] = $parent;
}
}
if (empty($discover)) {
break;
}
}
$n = count($insert);
if ($autoclose) {
$this->log("Found {$n} new autoclose commits on branch '{$branch}'.");
} else {
$this->log("Found {$n} new commits on branch '{$branch}'.");
}
while (true) {
$target = array_pop($insert);
$epoch = $stream->getCommitDate($target);
$epoch = trim($epoch);
if ($autoclose) {
$this->updateCommit($repository, $target, $branch);
} else {
$this->recordCommit($repository, $target, $epoch, $branch);
}
if (empty($insert)) {
break;
}
}
}
/**
* Verify that the "origin" remote exists, and points at the correct URI.
*
* This catches or corrects some types of misconfiguration, and also repairs
* an issue where Git 1.7.1 does not create an "origin" for `--bare` clones.
* See T4041.
*
* @param PhabricatorRepository Repository to verify.
* @return void
*/
private function verifyOrigin(PhabricatorRepository $repository) {
list($remotes) = $repository->execxLocalCommand(
'remote show -n origin');
$matches = null;
if (!preg_match('/^\s*Fetch URL:\s*(.*?)\s*$/m', $remotes, $matches)) {
throw new Exception(
"Expected 'Fetch URL' in 'git remote show -n origin'.");
}
$remote_uri = $matches[1];
$expect_remote = $repository->getRemoteURI();
if ($remote_uri == "origin") {
// If a remote does not exist, git pretends it does and prints out a
// made up remote where the URI is the same as the remote name. This is
// definitely not correct.
// Possibly, we should use `git remote --verbose` instead, which does not
// suffer from this problem (but is a little more complicated to parse).
$valid = false;
$exists = false;
} else {
$valid = self::isSameGitOrigin($remote_uri, $expect_remote);
$exists = true;
}
if (!$valid) {
if (!$exists) {
// If there's no "origin" remote, just create it regardless of how
// strongly we own the working copy. There is almost no conceivable
// scenario in which this could do damage.
$this->log(
pht(
'Remote "origin" does not exist. Creating "origin", with '.
'URI "%s".',
$expect_remote));
$repository->execxLocalCommand(
- 'remote add origin %s',
- $expect_remote);
+ 'remote add origin %P',
+ $repository->getRemoteURIEnvelope());
// NOTE: This doesn't fetch the origin (it just creates it), so we won't
// know about origin branches until the next "pull" happens. That's fine
// for our purposes, but might impact things in the future.
} else {
if ($repository->canDestroyWorkingCopy()) {
// Bad remote, but we can try to repair it.
$this->log(
pht(
'Remote "origin" exists, but is pointed at the wrong URI, "%s". '.
'Resetting origin URI to "%s.',
$remote_uri,
$expect_remote));
$repository->execxLocalCommand(
- 'remote set-url origin %s',
- $expect_remote);
+ 'remote set-url origin %P',
+ $repository->getRemoteURIEnvelope());
} else {
// Bad remote and we aren't comfortable repairing it.
$message = pht(
'Working copy at "%s" has a mismatched origin URI, "%s". '.
'The expected origin URI is "%s". Fix your configuration, or '.
'set the remote URI correctly. To avoid breaking anything, '.
'Phabricator will not automatically fix this.',
$repository->getLocalPath(),
$remote_uri,
$expect_remote);
throw new Exception($message);
}
}
}
}
/**
* @task git
*/
public static function isSameGitOrigin($remote, $expect) {
$remote_path = self::getPathFromGitURI($remote);
$expect_path = self::getPathFromGitURI($expect);
$remote_match = self::executeGitNormalizePath($remote_path);
$expect_match = self::executeGitNormalizePath($expect_path);
return ($remote_match == $expect_match);
}
private static function getPathFromGitURI($raw_uri) {
$uri = new PhutilURI($raw_uri);
if ($uri->getProtocol()) {
return $uri->getPath();
}
$uri = new PhutilGitURI($raw_uri);
if ($uri->getDomain()) {
return $uri->getPath();
}
return $raw_uri;
}
/**
* @task git
*/
private static function executeGitNormalizePath($path) {
// Strip away "/" and ".git", so similar paths correctly match.
$path = trim($path, '/');
$path = preg_replace('/\.git$/', '', $path);
return $path;
}
private function pushToMirrors(PhabricatorRepository $repository) {
if (!$repository->canMirror()) {
return;
}
$mirrors = id(new PhabricatorRepositoryMirrorQuery())
->setViewer($this->getViewer())
->withRepositoryPHIDs(array($repository->getPHID()))
->execute();
// TODO: This is a little bit janky, but we don't have first-class
// infrastructure for running remote commands against an arbitrary remote
// right now. Just make an emphemeral copy of the repository and muck with
// it a little bit. In the medium term, we should pull this command stuff
// out and use it here and for "Land to ...".
$proxy = clone $repository;
$proxy->makeEphemeral();
$proxy->setDetail('hosting-enabled', false);
foreach ($mirrors as $mirror) {
$proxy->setDetail('remote-uri', $mirror->getRemoteURI());
$proxy->setCredentialPHID($mirror->getCredentialPHID());
$this->log(pht('Pushing to remote "%s"...', $mirror->getRemoteURI()));
if (!$proxy->isGit()) {
throw new Exception('Unsupported VCS!');
}
$future = $proxy->getRemoteCommandFuture(
'push --verbose --mirror -- %P',
$proxy->getRemoteURIEnvelope());
$future
->setCWD($proxy->getLocalPath())
->resolvex();
}
}
}

File Metadata

Mime Type
text/x-diff
Expires
Thu, Nov 13, 3:48 PM (1 h, 47 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
336253
Default Alt Text
(51 KB)

Event Timeline