Page MenuHomestyx hydra
Files F127627

No OneTemporary
Actions

View Options

diff --git a/src/applications/diffusion/conduit/DiffusionBranchQueryConduitAPIMethod.php b/src/applications/diffusion/conduit/DiffusionBranchQueryConduitAPIMethod.php
index 5fd440e4b2..d42c45a96b 100644
--- a/src/applications/diffusion/conduit/DiffusionBranchQueryConduitAPIMethod.php
+++ b/src/applications/diffusion/conduit/DiffusionBranchQueryConduitAPIMethod.php
@@ -1,125 +1,128 @@
<?php
final class DiffusionBranchQueryConduitAPIMethod
extends DiffusionQueryConduitAPIMethod {
public function getAPIMethodName() {
return 'diffusion.branchquery';
}
public function getMethodDescription() {
return pht('Determine what branches exist for a repository.');
}
protected function defineReturnType() {
return 'list<dict>';
}
protected function defineCustomParamTypes() {
return array(
'closed' => 'optional bool',
'limit' => 'optional int',
'offset' => 'optional int',
'contains' => 'optional string',
);
}
protected function getGitResult(ConduitAPIRequest $request) {
$drequest = $this->getDiffusionRequest();
$repository = $drequest->getRepository();
$contains = $request->getValue('contains');
if (strlen($contains)) {
// NOTE: We can't use DiffusionLowLevelGitRefQuery here because
// `git for-each-ref` does not support `--contains`.
if ($repository->isWorkingCopyBare()) {
list($stdout) = $repository->execxLocalCommand(
'branch --verbose --no-abbrev --contains %s --',
$contains);
$ref_map = DiffusionGitBranch::parseLocalBranchOutput(
$stdout);
} else {
list($stdout) = $repository->execxLocalCommand(
'branch -r --verbose --no-abbrev --contains %s --',
$contains);
$ref_map = DiffusionGitBranch::parseRemoteBranchOutput(
$stdout,
DiffusionGitBranch::DEFAULT_GIT_REMOTE);
}
$refs = array();
foreach ($ref_map as $ref => $commit) {
$refs[] = id(new DiffusionRepositoryRef())
->setShortName($ref)
->setCommitIdentifier($commit);
}
} else {
$refs = id(new DiffusionLowLevelGitRefQuery())
->setRepository($repository)
- ->withIsOriginBranch(true)
+ ->withRefTypes(
+ array(
+ PhabricatorRepositoryRefCursor::TYPE_BRANCH,
+ ))
->execute();
}
return $this->processBranchRefs($request, $refs);
}
protected function getMercurialResult(ConduitAPIRequest $request) {
$drequest = $this->getDiffusionRequest();
$repository = $drequest->getRepository();
$query = id(new DiffusionLowLevelMercurialBranchesQuery())
->setRepository($repository);
$contains = $request->getValue('contains');
if (strlen($contains)) {
$query->withContainsCommit($contains);
}
$refs = $query->execute();
return $this->processBranchRefs($request, $refs);
}
protected function getSVNResult(ConduitAPIRequest $request) {
// Since SVN doesn't have meaningful branches, just return nothing for all
// queries.
return array();
}
private function processBranchRefs(ConduitAPIRequest $request, array $refs) {
$drequest = $this->getDiffusionRequest();
$repository = $drequest->getRepository();
$offset = $request->getValue('offset');
$limit = $request->getValue('limit');
foreach ($refs as $key => $ref) {
if (!$repository->shouldTrackBranch($ref->getShortName())) {
unset($refs[$key]);
}
}
$with_closed = $request->getValue('closed');
if ($with_closed !== null) {
foreach ($refs as $key => $ref) {
$fields = $ref->getRawFields();
if (idx($fields, 'closed') != $with_closed) {
unset($refs[$key]);
}
}
}
// NOTE: We can't apply the offset or limit until here, because we may have
// filtered untrackable branches out of the result set.
if ($offset) {
$refs = array_slice($refs, $offset);
}
if ($limit) {
$refs = array_slice($refs, 0, $limit);
}
return mpull($refs, 'toDictionary');
}
}
diff --git a/src/applications/diffusion/conduit/DiffusionTagsQueryConduitAPIMethod.php b/src/applications/diffusion/conduit/DiffusionTagsQueryConduitAPIMethod.php
index ddf3a2152b..3de5793289 100644
--- a/src/applications/diffusion/conduit/DiffusionTagsQueryConduitAPIMethod.php
+++ b/src/applications/diffusion/conduit/DiffusionTagsQueryConduitAPIMethod.php
@@ -1,168 +1,171 @@
<?php
final class DiffusionTagsQueryConduitAPIMethod
extends DiffusionQueryConduitAPIMethod {
public function getAPIMethodName() {
return 'diffusion.tagsquery';
}
public function getMethodDescription() {
return pht('Retrieve information about tags in a repository.');
}
protected function defineReturnType() {
return 'array';
}
protected function defineCustomParamTypes() {
return array(
'names' => 'optional list<string>',
'commit' => 'optional string',
'needMessages' => 'optional bool',
'offset' => 'optional int',
'limit' => 'optional int',
);
}
protected function getGitResult(ConduitAPIRequest $request) {
$drequest = $this->getDiffusionRequest();
$repository = $drequest->getRepository();
$commit = $drequest->getSymbolicCommit();
$commit_filter = null;
if ($commit) {
$commit_filter = $this->loadTagNamesForCommit($commit);
}
$name_filter = $request->getValue('names', null);
$all_tags = $this->loadGitTagList();
$all_tags = mpull($all_tags, null, 'getName');
if ($name_filter !== null) {
$all_tags = array_intersect_key($all_tags, array_fuse($name_filter));
}
if ($commit_filter !== null) {
$all_tags = array_intersect_key($all_tags, $commit_filter);
}
$tags = array_values($all_tags);
$offset = $request->getValue('offset');
$limit = $request->getValue('limit');
if ($offset) {
$tags = array_slice($tags, $offset);
}
if ($limit) {
$tags = array_slice($tags, 0, $limit);
}
if ($request->getValue('needMessages')) {
$this->loadMessagesForTags($all_tags);
}
return mpull($tags, 'toDictionary');
}
private function loadGitTagList() {
$drequest = $this->getDiffusionRequest();
$repository = $drequest->getRepository();
$refs = id(new DiffusionLowLevelGitRefQuery())
->setRepository($repository)
- ->withIsTag(true)
+ ->withRefTypes(
+ array(
+ PhabricatorRepositoryRefCursor::TYPE_TAG,
+ ))
->execute();
$tags = array();
foreach ($refs as $ref) {
$fields = $ref->getRawFields();
$tag = id(new DiffusionRepositoryTag())
->setAuthor($fields['author'])
->setEpoch($fields['epoch'])
->setCommitIdentifier($ref->getCommitIdentifier())
->setName($ref->getShortName())
->setDescription($fields['subject'])
->setType('git/'.$fields['objecttype']);
$tags[] = $tag;
}
return $tags;
}
private function loadTagNamesForCommit($commit) {
$drequest = $this->getDiffusionRequest();
$repository = $drequest->getRepository();
list($err, $stdout) = $repository->execLocalCommand(
'tag -l --contains %s',
$commit);
if ($err) {
// Git exits with an error code if the commit is bogus.
return array();
}
$stdout = rtrim($stdout, "\n");
if (!strlen($stdout)) {
return array();
}
$tag_names = explode("\n", $stdout);
$tag_names = array_fill_keys($tag_names, true);
return $tag_names;
}
private function loadMessagesForTags(array $tags) {
assert_instances_of($tags, 'DiffusionRepositoryTag');
$drequest = $this->getDiffusionRequest();
$repository = $drequest->getRepository();
$futures = array();
foreach ($tags as $key => $tag) {
$futures[$key] = $repository->getLocalCommandFuture(
'cat-file tag %s',
$tag->getName());
}
id(new FutureIterator($futures))
->resolveAll();
foreach ($tags as $key => $tag) {
$future = $futures[$key];
list($err, $stdout) = $future->resolve();
$message = null;
if ($err) {
// Not all tags are actually "tag" objects: a "tag" object is only
// created if you provide a message or sign the tag. Tags created with
// `git tag x [commit]` are "lightweight tags" and `git cat-file tag`
// will fail on them. This is fine: they don't have messages.
} else {
$parts = explode("\n\n", $stdout, 2);
if (count($parts) == 2) {
$message = last($parts);
}
}
$tag->attachMessage($message);
}
return $tags;
}
protected function getMercurialResult(ConduitAPIRequest $request) {
// For now, we don't support Mercurial tags via API.
return array();
}
protected function getSVNResult(ConduitAPIRequest $request) {
// Subversion has no meaningful concept of tags.
return array();
}
}
diff --git a/src/applications/diffusion/editor/DiffusionURIEditor.php b/src/applications/diffusion/editor/DiffusionURIEditor.php
index 020275a367..c22b888ac5 100644
--- a/src/applications/diffusion/editor/DiffusionURIEditor.php
+++ b/src/applications/diffusion/editor/DiffusionURIEditor.php
@@ -1,483 +1,496 @@
<?php
final class DiffusionURIEditor
extends PhabricatorApplicationTransactionEditor {
private $repository;
private $repositoryPHID;
public function getEditorApplicationClass() {
return 'PhabricatorDiffusionApplication';
}
public function getEditorObjectsDescription() {
return pht('Diffusion URIs');
}
public function getTransactionTypes() {
$types = parent::getTransactionTypes();
$types[] = PhabricatorRepositoryURITransaction::TYPE_REPOSITORY;
$types[] = PhabricatorRepositoryURITransaction::TYPE_URI;
$types[] = PhabricatorRepositoryURITransaction::TYPE_IO;
$types[] = PhabricatorRepositoryURITransaction::TYPE_DISPLAY;
$types[] = PhabricatorRepositoryURITransaction::TYPE_CREDENTIAL;
$types[] = PhabricatorRepositoryURITransaction::TYPE_DISABLE;
return $types;
}
protected function getCustomTransactionOldValue(
PhabricatorLiskDAO $object,
PhabricatorApplicationTransaction $xaction) {
switch ($xaction->getTransactionType()) {
case PhabricatorRepositoryURITransaction::TYPE_URI:
return $object->getURI();
case PhabricatorRepositoryURITransaction::TYPE_IO:
return $object->getIOType();
case PhabricatorRepositoryURITransaction::TYPE_DISPLAY:
return $object->getDisplayType();
case PhabricatorRepositoryURITransaction::TYPE_REPOSITORY:
return $object->getRepositoryPHID();
case PhabricatorRepositoryURITransaction::TYPE_CREDENTIAL:
return $object->getCredentialPHID();
case PhabricatorRepositoryURITransaction::TYPE_DISABLE:
return (int)$object->getIsDisabled();
}
return parent::getCustomTransactionOldValue($object, $xaction);
}
protected function getCustomTransactionNewValue(
PhabricatorLiskDAO $object,
PhabricatorApplicationTransaction $xaction) {
switch ($xaction->getTransactionType()) {
case PhabricatorRepositoryURITransaction::TYPE_URI:
case PhabricatorRepositoryURITransaction::TYPE_IO:
case PhabricatorRepositoryURITransaction::TYPE_DISPLAY:
case PhabricatorRepositoryURITransaction::TYPE_REPOSITORY:
case PhabricatorRepositoryURITransaction::TYPE_CREDENTIAL:
return $xaction->getNewValue();
case PhabricatorRepositoryURITransaction::TYPE_DISABLE:
return (int)$xaction->getNewValue();
}
return parent::getCustomTransactionNewValue($object, $xaction);
}
protected function applyCustomInternalTransaction(
PhabricatorLiskDAO $object,
PhabricatorApplicationTransaction $xaction) {
switch ($xaction->getTransactionType()) {
case PhabricatorRepositoryURITransaction::TYPE_URI:
if (!$this->getIsNewObject()) {
$old_uri = $object->getEffectiveURI();
} else {
$old_uri = null;
}
$object->setURI($xaction->getNewValue());
// If we've changed the domain or protocol of the URI, remove the
// current credential. This improves behavior in several cases:
// If a user switches between protocols with different credential
// types, like HTTP and SSH, the old credential won't be valid anyway.
// It's cleaner to remove it than leave a bad credential in place.
// If a user switches hosts, the old credential is probably not
// correct (and potentially confusing/misleading). Removing it forces
// users to double check that they have the correct credentials.
// If an attacker can't see a symmetric credential like a username and
// password, they could still potentially capture it by changing the
// host for a URI that uses it to `evil.com`, a server they control,
// then observing the requests. Removing the credential prevents this
// kind of escalation.
// Since port and path changes are less likely to fall among these
// cases, they don't trigger a credential wipe.
$new_uri = $object->getEffectiveURI();
if ($old_uri) {
$new_proto = ($old_uri->getProtocol() != $new_uri->getProtocol());
$new_domain = ($old_uri->getDomain() != $new_uri->getDomain());
if ($new_proto || $new_domain) {
$object->setCredentialPHID(null);
}
}
break;
case PhabricatorRepositoryURITransaction::TYPE_IO:
$object->setIOType($xaction->getNewValue());
break;
case PhabricatorRepositoryURITransaction::TYPE_DISPLAY:
$object->setDisplayType($xaction->getNewValue());
break;
case PhabricatorRepositoryURITransaction::TYPE_REPOSITORY:
$object->setRepositoryPHID($xaction->getNewValue());
$object->attachRepository($this->repository);
break;
case PhabricatorRepositoryURITransaction::TYPE_CREDENTIAL:
$object->setCredentialPHID($xaction->getNewValue());
break;
case PhabricatorRepositoryURITransaction::TYPE_DISABLE:
$object->setIsDisabled($xaction->getNewValue());
break;
}
}
protected function applyCustomExternalTransaction(
PhabricatorLiskDAO $object,
PhabricatorApplicationTransaction $xaction) {
switch ($xaction->getTransactionType()) {
case PhabricatorRepositoryURITransaction::TYPE_URI:
case PhabricatorRepositoryURITransaction::TYPE_IO:
case PhabricatorRepositoryURITransaction::TYPE_DISPLAY:
case PhabricatorRepositoryURITransaction::TYPE_REPOSITORY:
case PhabricatorRepositoryURITransaction::TYPE_CREDENTIAL:
case PhabricatorRepositoryURITransaction::TYPE_DISABLE:
return;
}
return parent::applyCustomExternalTransaction($object, $xaction);
}
protected function validateTransaction(
PhabricatorLiskDAO $object,
$type,
array $xactions) {
$errors = parent::validateTransaction($object, $type, $xactions);
switch ($type) {
case PhabricatorRepositoryURITransaction::TYPE_REPOSITORY:
// Save this, since we need it to validate TYPE_IO transactions.
$this->repositoryPHID = $object->getRepositoryPHID();
$missing = $this->validateIsEmptyTextField(
$object->getRepositoryPHID(),
$xactions);
if ($missing) {
// NOTE: This isn't being marked as a missing field error because
// it's a fundamental, required property of the URI.
$errors[] = new PhabricatorApplicationTransactionValidationError(
$type,
pht('Required'),
pht(
'When creating a repository URI, you must specify which '.
'repository the URI will belong to.'),
nonempty(last($xactions), null));
break;
}
$viewer = $this->getActor();
foreach ($xactions as $xaction) {
$repository_phid = $xaction->getNewValue();
// If this isn't changing anything, let it through as-is.
if ($repository_phid == $object->getRepositoryPHID()) {
continue;
}
if (!$this->getIsNewObject()) {
$errors[] = new PhabricatorApplicationTransactionValidationError(
$type,
pht('Invalid'),
pht(
'The repository a URI is associated with is immutable, and '.
'can not be changed after the URI is created.'),
$xaction);
continue;
}
$repository = id(new PhabricatorRepositoryQuery())
->setViewer($viewer)
->withPHIDs(array($repository_phid))
->requireCapabilities(
array(
PhabricatorPolicyCapability::CAN_VIEW,
PhabricatorPolicyCapability::CAN_EDIT,
))
->executeOne();
if (!$repository) {
$errors[] = new PhabricatorApplicationTransactionValidationError(
$type,
pht('Invalid'),
pht(
'To create a URI for a repository ("%s"), it must exist and '.
'you must have permission to edit it.',
$repository_phid),
$xaction);
continue;
}
$this->repository = $repository;
$this->repositoryPHID = $repository_phid;
}
break;
case PhabricatorRepositoryURITransaction::TYPE_CREDENTIAL:
$viewer = $this->getActor();
foreach ($xactions as $xaction) {
$credential_phid = $xaction->getNewValue();
if ($credential_phid == $object->getCredentialPHID()) {
continue;
}
// Anyone who can edit a URI can remove the credential.
if ($credential_phid === null) {
continue;
}
$credential = id(new PassphraseCredentialQuery())
->setViewer($viewer)
->withPHIDs(array($credential_phid))
->executeOne();
if (!$credential) {
$errors[] = new PhabricatorApplicationTransactionValidationError(
$type,
pht('Invalid'),
pht(
'You can only associate a credential ("%s") with a repository '.
'URI if it exists and you have permission to see it.',
$credential_phid),
$xaction);
continue;
}
}
break;
case PhabricatorRepositoryURITransaction::TYPE_URI:
$missing = $this->validateIsEmptyTextField(
$object->getURI(),
$xactions);
if ($missing) {
$error = new PhabricatorApplicationTransactionValidationError(
$type,
pht('Required'),
pht('A repository URI must have a nonempty URI.'),
nonempty(last($xactions), null));
$error->setIsMissingFieldError(true);
$errors[] = $error;
break;
}
foreach ($xactions as $xaction) {
$new_uri = $xaction->getNewValue();
if ($new_uri == $object->getURI()) {
continue;
}
try {
PhabricatorRepository::assertValidRemoteURI($new_uri);
} catch (Exception $ex) {
$errors[] = new PhabricatorApplicationTransactionValidationError(
$type,
pht('Invalid'),
$ex->getMessage(),
$xaction);
continue;
}
}
break;
case PhabricatorRepositoryURITransaction::TYPE_IO:
$available = $object->getAvailableIOTypeOptions();
foreach ($xactions as $xaction) {
$new = $xaction->getNewValue();
if (empty($available[$new])) {
$errors[] = new PhabricatorApplicationTransactionValidationError(
$type,
pht('Invalid'),
pht(
'Value "%s" is not a valid display setting for this URI. '.
'Available types for this URI are: %s.',
implode(', ', array_keys($available))),
$xaction);
continue;
}
// If we are setting this URI to use "Observe", we must have no
// other "Observe" URIs and must also have no "Read/Write" URIs.
// If we are setting this URI to "Read/Write", we must have no
// other "Observe" URIs. It's OK to have other "Read/Write" URIs.
$no_observers = false;
$no_readwrite = false;
switch ($new) {
case PhabricatorRepositoryURI::IO_OBSERVE:
$no_readwrite = true;
$no_observers = true;
break;
case PhabricatorRepositoryURI::IO_READWRITE:
$no_observers = true;
break;
}
if ($no_observers || $no_readwrite) {
$repository = id(new PhabricatorRepositoryQuery())
->setViewer(PhabricatorUser::getOmnipotentUser())
->withPHIDs(array($this->repositoryPHID))
->needURIs(true)
->executeOne();
$uris = $repository->getURIs();
$observe_conflict = null;
$readwrite_conflict = null;
foreach ($uris as $uri) {
// If this is the URI being edited, it can not conflict with
// itself.
if ($uri->getID() == $object->getID()) {
continue;
}
$io_type = $uri->getIoType();
if ($io_type == PhabricatorRepositoryURI::IO_READWRITE) {
if ($no_readwrite) {
$readwite_conflict = $uri;
break;
}
}
if ($io_type == PhabricatorRepositoryURI::IO_OBSERVE) {
if ($no_observers) {
$observe_conflict = $uri;
break;
}
}
}
if ($observe_conflict) {
if ($new == PhabricatorRepositoryURI::IO_OBSERVE) {
$message = pht(
'You can not set this URI to use Observe IO because '.
'another URI for this repository is already configured '.
'in Observe IO mode. A repository can not observe two '.
'different remotes simultaneously. Turn off IO for the '.
'other URI first.');
} else {
$message = pht(
'You can not set this URI to use Read/Write IO because '.
'another URI for this repository is already configured '.
'in Observe IO mode. An observed repository can not be '.
'made writable. Turn off IO for the other URI first.');
}
$errors[] = new PhabricatorApplicationTransactionValidationError(
$type,
pht('Invalid'),
$message,
$xaction);
continue;
}
if ($readwrite_conflict) {
$message = pht(
'You can not set this URI to use Observe IO because '.
'another URI for this repository is already configured '.
'in Read/Write IO mode. A repository can not simultaneously '.
'be writable and observe a remote. Turn off IO for the '.
'other URI first.');
$errors[] = new PhabricatorApplicationTransactionValidationError(
$type,
pht('Invalid'),
$message,
$xaction);
continue;
}
}
}
break;
case PhabricatorRepositoryURITransaction::TYPE_DISPLAY:
$available = $object->getAvailableDisplayTypeOptions();
foreach ($xactions as $xaction) {
$new = $xaction->getNewValue();
if (empty($available[$new])) {
$errors[] = new PhabricatorApplicationTransactionValidationError(
$type,
pht('Invalid'),
pht(
'Value "%s" is not a valid display setting for this URI. '.
'Available types for this URI are: %s.',
implode(', ', array_keys($available))));
}
}
break;
case PhabricatorRepositoryURITransaction::TYPE_DISABLE:
$old = $object->getIsDisabled();
foreach ($xactions as $xaction) {
$new = $xaction->getNewValue();
if ($old == $new) {
continue;
}
if (!$object->isBuiltin()) {
continue;
}
$errors[] = new PhabricatorApplicationTransactionValidationError(
$type,
pht('Invalid'),
pht('You can not manually disable builtin URIs.'));
}
break;
}
return $errors;
}
protected function applyFinalEffects(
PhabricatorLiskDAO $object,
array $xactions) {
// Synchronize the repository state based on the presence of an "Observe"
// URI.
$repository = $object->getRepository();
$uris = id(new PhabricatorRepositoryURIQuery())
->setViewer(PhabricatorUser::getOmnipotentUser())
->withRepositories(array($repository))
->execute();
$observe_uri = null;
foreach ($uris as $uri) {
if ($uri->getIoType() != PhabricatorRepositoryURI::IO_OBSERVE) {
continue;
}
$observe_uri = $uri;
break;
}
+ $was_hosted = $repository->isHosted();
+
if ($observe_uri) {
$repository
->setHosted(false)
->setDetail('remote-uri', (string)$observe_uri->getEffectiveURI())
->setCredentialPHID($observe_uri->getCredentialPHID());
} else {
$repository
->setHosted(true)
->setDetail('remote-uri', null)
->setCredentialPHID(null);
}
$repository->save();
+ $is_hosted = $repository->isHosted();
+
+ // If we've swapped the repository from hosted to observed or vice versa,
+ // reset all the cluster version clocks.
+ if ($was_hosted != $is_hosted) {
+ $cluster_engine = id(new DiffusionRepositoryClusterEngine())
+ ->setViewer($this->getActor())
+ ->setRepository($repository)
+ ->synchronizeWorkingCopyAfterHostingChange();
+ }
+
return $xactions;
}
}
diff --git a/src/applications/diffusion/management/DiffusionRepositoryStorageManagementPanel.php b/src/applications/diffusion/management/DiffusionRepositoryStorageManagementPanel.php
index 0c723bd9e6..9cf210632d 100644
--- a/src/applications/diffusion/management/DiffusionRepositoryStorageManagementPanel.php
+++ b/src/applications/diffusion/management/DiffusionRepositoryStorageManagementPanel.php
@@ -1,235 +1,251 @@
<?php
final class DiffusionRepositoryStorageManagementPanel
extends DiffusionRepositoryManagementPanel {
const PANELKEY = 'storage';
public function getManagementPanelLabel() {
return pht('Storage');
}
public function getManagementPanelOrder() {
return 600;
}
public function getManagementPanelIcon() {
$repository = $this->getRepository();
if ($repository->getAlmanacServicePHID()) {
return 'fa-sitemap';
} else if ($repository->isHosted()) {
return 'fa-folder';
} else {
return 'fa-download';
}
}
public function buildManagementPanelContent() {
return array(
$this->buildStorageStatusPanel(),
$this->buildClusterStatusPanel(),
);
}
private function buildStorageStatusPanel() {
$repository = $this->getRepository();
$viewer = $this->getViewer();
$view = id(new PHUIPropertyListView())
->setViewer($viewer);
if ($repository->usesLocalWorkingCopy()) {
$storage_path = $repository->getLocalPath();
} else {
$storage_path = phutil_tag('em', array(), pht('No Local Working Copy'));
}
$service_phid = $repository->getAlmanacServicePHID();
if ($service_phid) {
$storage_service = $viewer->renderHandle($service_phid);
} else {
$storage_service = phutil_tag('em', array(), pht('Local'));
}
$view->addProperty(pht('Storage Path'), $storage_path);
$view->addProperty(pht('Storage Cluster'), $storage_service);
$header = id(new PHUIHeaderView())
->setHeader(pht('Storage'));
return id(new PHUIObjectBoxView())
->setHeader($header)
->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
->addPropertyList($view);
}
private function buildClusterStatusPanel() {
$repository = $this->getRepository();
$viewer = $this->getViewer();
$service_phid = $repository->getAlmanacServicePHID();
if ($service_phid) {
$service = id(new AlmanacServiceQuery())
->setViewer($viewer)
->withServiceTypes(
array(
AlmanacClusterRepositoryServiceType::SERVICETYPE,
))
->withPHIDs(array($service_phid))
->needBindings(true)
->executeOne();
if (!$service) {
// TODO: Viewer may not have permission to see the service, or it may
// be invalid? Raise some more useful error here?
throw new Exception(pht('Unable to load cluster service.'));
}
} else {
$service = null;
}
Javelin::initBehavior('phabricator-tooltips');
$rows = array();
if ($service) {
$bindings = $service->getBindings();
$bindings = mgroup($bindings, 'getDevicePHID');
// This is an unusual read which always comes from the master.
if (PhabricatorEnv::isReadOnly()) {
$versions = array();
} else {
$versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions(
$repository->getPHID());
}
$versions = mpull($versions, null, 'getDevicePHID');
foreach ($bindings as $binding_group) {
$all_disabled = true;
foreach ($binding_group as $binding) {
if (!$binding->getIsDisabled()) {
$all_disabled = false;
break;
}
}
$any_binding = head($binding_group);
if ($all_disabled) {
$binding_icon = 'fa-times grey';
$binding_tip = pht('Disabled');
} else {
$binding_icon = 'fa-folder-open green';
$binding_tip = pht('Active');
}
$binding_icon = id(new PHUIIconView())
->setIcon($binding_icon)
->addSigil('has-tooltip')
->setMetadata(
array(
'tip' => $binding_tip,
));
$device = $any_binding->getDevice();
$version = idx($versions, $device->getPHID());
if ($version) {
$version_number = $version->getRepositoryVersion();
- $version_number = phutil_tag(
- 'a',
- array(
- 'href' => "/diffusion/pushlog/view/{$version_number}/",
- ),
- $version_number);
+
+ $href = null;
+ if ($repository->isHosted()) {
+ $href = "/diffusion/pushlog/view/{$version_number}/";
+ } else {
+ $commit = id(new DiffusionCommitQuery())
+ ->setViewer($viewer)
+ ->withIDs(array($version_number))
+ ->executeOne();
+ if ($commit) {
+ $href = $commit->getURI();
+ }
+ }
+
+ if ($href) {
+ $version_number = phutil_tag(
+ 'a',
+ array(
+ 'href' => $href,
+ ),
+ $version_number);
+ }
} else {
$version_number = '-';
}
if ($version && $version->getIsWriting()) {
$is_writing = id(new PHUIIconView())
->setIcon('fa-pencil green');
} else {
$is_writing = id(new PHUIIconView())
->setIcon('fa-pencil grey');
}
$write_properties = null;
if ($version) {
$write_properties = $version->getWriteProperties();
if ($write_properties) {
try {
$write_properties = phutil_json_decode($write_properties);
} catch (Exception $ex) {
$write_properties = null;
}
}
}
if ($write_properties) {
$writer_phid = idx($write_properties, 'userPHID');
$last_writer = $viewer->renderHandle($writer_phid);
$writer_epoch = idx($write_properties, 'epoch');
$writer_epoch = phabricator_datetime($writer_epoch, $viewer);
} else {
$last_writer = null;
$writer_epoch = null;
}
$rows[] = array(
$binding_icon,
phutil_tag(
'a',
array(
'href' => $device->getURI(),
),
$device->getName()),
$version_number,
$is_writing,
$last_writer,
$writer_epoch,
);
}
}
$table = id(new AphrontTableView($rows))
->setNoDataString(pht('This is not a cluster repository.'))
->setHeaders(
array(
null,
pht('Device'),
pht('Version'),
pht('Writing'),
pht('Last Writer'),
pht('Last Write At'),
))
->setColumnClasses(
array(
null,
null,
null,
'right wide',
null,
'date',
));
$doc_href = PhabricatorEnv::getDoclink('Cluster: Repositories');
$header = id(new PHUIHeaderView())
->setHeader(pht('Cluster Status'))
->addActionLink(
id(new PHUIButtonView())
->setIcon('fa-book')
->setHref($doc_href)
->setTag('a')
->setText(pht('Documentation')));
return id(new PHUIObjectBoxView())
->setHeader($header)
->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
->setTable($table);
}
}
diff --git a/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php b/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php
index b271c16741..fad8d4019e 100644
--- a/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php
+++ b/src/applications/diffusion/protocol/DiffusionRepositoryClusterEngine.php
@@ -1,625 +1,764 @@
<?php
/**
* Manages repository synchronization for cluster repositories.
*
* @task config Configuring Synchronization
* @task sync Cluster Synchronization
* @task internal Internals
*/
final class DiffusionRepositoryClusterEngine extends Phobject {
private $repository;
private $viewer;
private $logger;
private $clusterWriteLock;
private $clusterWriteVersion;
private $clusterWriteOwner;
/* -( Configuring Synchronization )---------------------------------------- */
public function setRepository(PhabricatorRepository $repository) {
$this->repository = $repository;
return $this;
}
public function getRepository() {
return $this->repository;
}
public function setViewer(PhabricatorUser $viewer) {
$this->viewer = $viewer;
return $this;
}
public function getViewer() {
return $this->viewer;
}
public function setLog(DiffusionRepositoryClusterEngineLogInterface $log) {
$this->logger = $log;
return $this;
}
/* -( Cluster Synchronization )-------------------------------------------- */
/**
* Synchronize repository version information after creating a repository.
*
* This initializes working copy versions for all currently bound devices to
* 0, so that we don't get stuck making an ambiguous choice about which
* devices are leaders when we later synchronize before a read.
*
* @task sync
*/
public function synchronizeWorkingCopyAfterCreation() {
if (!$this->shouldEnableSynchronization()) {
return;
}
$repository = $this->getRepository();
$repository_phid = $repository->getPHID();
$service = $repository->loadAlmanacService();
if (!$service) {
throw new Exception(pht('Failed to load repository cluster service.'));
}
$bindings = $service->getActiveBindings();
foreach ($bindings as $binding) {
PhabricatorRepositoryWorkingCopyVersion::updateVersion(
$repository_phid,
$binding->getDevicePHID(),
0);
}
return $this;
}
+ /**
+ * @task sync
+ */
+ public function synchronizeWorkingCopyAfterHostingChange() {
+ if (!$this->shouldEnableSynchronization()) {
+ return;
+ }
+
+ $repository = $this->getRepository();
+ $repository_phid = $repository->getPHID();
+
+ $versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions(
+ $repository_phid);
+ $versions = mpull($versions, null, 'getDevicePHID');
+
+ // After converting a hosted repository to observed, or vice versa, we
+ // need to reset version numbers because the clocks for observed and hosted
+ // repositories run on different units.
+
+ // We identify all the cluster leaders and reset their version to 0.
+ // We identify all the cluster followers and demote them.
+
+ // This allows the cluter to start over again at version 0 but keep the
+ // same leaders.
+
+ if ($versions) {
+ $max_version = (int)max(mpull($versions, 'getRepositoryVersion'));
+ foreach ($versions as $version) {
+ $device_phid = $version->getDevicePHID();
+
+ if ($version->getRepositoryVersion() == $max_version) {
+ PhabricatorRepositoryWorkingCopyVersion::updateVersion(
+ $repository_phid,
+ $device_phid,
+ 0);
+ } else {
+ PhabricatorRepositoryWorkingCopyVersion::demoteDevice(
+ $repository_phid,
+ $device_phid);
+ }
+ }
+ }
+
+ return $this;
+ }
+
+
/**
* @task sync
*/
public function synchronizeWorkingCopyBeforeRead() {
if (!$this->shouldEnableSynchronization()) {
return;
}
$repository = $this->getRepository();
$repository_phid = $repository->getPHID();
$device = AlmanacKeys::getLiveDevice();
$device_phid = $device->getPHID();
$read_lock = PhabricatorRepositoryWorkingCopyVersion::getReadLock(
$repository_phid,
$device_phid);
$lock_wait = phutil_units('2 minutes in seconds');
$this->logLine(
pht(
'Waiting up to %s second(s) for a cluster read lock on "%s"...',
new PhutilNumber($lock_wait),
$device->getName()));
try {
$start = PhabricatorTime::getNow();
$read_lock->lock($lock_wait);
$waited = (PhabricatorTime::getNow() - $start);
if ($waited) {
$this->logLine(
pht(
'Acquired read lock after %s second(s).',
new PhutilNumber($waited)));
} else {
$this->logLine(
pht(
'Acquired read lock immediately.'));
}
} catch (Exception $ex) {
throw new PhutilProxyException(
pht(
'Failed to acquire read lock after waiting %s second(s). You '.
'may be able to retry later.',
new PhutilNumber($lock_wait)));
}
$versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions(
$repository_phid);
$versions = mpull($versions, null, 'getDevicePHID');
$this_version = idx($versions, $device_phid);
if ($this_version) {
$this_version = (int)$this_version->getRepositoryVersion();
} else {
$this_version = -1;
}
if ($versions) {
// This is the normal case, where we have some version information and
// can identify which nodes are leaders. If the current node is not a
// leader, we want to fetch from a leader and then update our version.
$max_version = (int)max(mpull($versions, 'getRepositoryVersion'));
if ($max_version > $this_version) {
- $fetchable = array();
- foreach ($versions as $version) {
- if ($version->getRepositoryVersion() == $max_version) {
- $fetchable[] = $version->getDevicePHID();
+ if ($repository->isHosted()) {
+ $fetchable = array();
+ foreach ($versions as $version) {
+ if ($version->getRepositoryVersion() == $max_version) {
+ $fetchable[] = $version->getDevicePHID();
+ }
}
- }
- $this->synchronizeWorkingCopyFromDevices($fetchable);
+ $this->synchronizeWorkingCopyFromDevices($fetchable);
+ } else {
+ $this->synchornizeWorkingCopyFromRemote();
+ }
PhabricatorRepositoryWorkingCopyVersion::updateVersion(
$repository_phid,
$device_phid,
$max_version);
} else {
$this->logLine(
pht(
'Device "%s" is already a cluster leader and does not need '.
'to be synchronized.',
$device->getName()));
}
$result_version = $max_version;
} else {
// If no version records exist yet, we need to be careful, because we
// can not tell which nodes are leaders.
// There might be several nodes with arbitrary existing data, and we have
// no way to tell which one has the "right" data. If we pick wrong, we
// might erase some or all of the data in the repository.
// Since this is dangeorus, we refuse to guess unless there is only one
// device. If we're the only device in the group, we obviously must be
// a leader.
$service = $repository->loadAlmanacService();
if (!$service) {
throw new Exception(pht('Failed to load repository cluster service.'));
}
$bindings = $service->getActiveBindings();
$device_map = array();
foreach ($bindings as $binding) {
$device_map[$binding->getDevicePHID()] = true;
}
if (count($device_map) > 1) {
throw new Exception(
pht(
'Repository "%s" exists on more than one device, but no device '.
'has any repository version information. Phabricator can not '.
'guess which copy of the existing data is authoritative. Remove '.
'all but one device from service to mark the remaining device '.
'as the authority.',
$repository->getDisplayName()));
}
if (empty($device_map[$device->getPHID()])) {
throw new Exception(
pht(
'Repository "%s" is being synchronized on device "%s", but '.
'this device is not bound to the corresponding cluster '.
'service ("%s").',
$repository->getDisplayName(),
$device->getName(),
$service->getName()));
}
// The current device is the only device in service, so it must be a
// leader. We can safely have any future nodes which come online read
// from it.
PhabricatorRepositoryWorkingCopyVersion::updateVersion(
$repository_phid,
$device_phid,
0);
$result_version = 0;
}
$read_lock->unlock();
return $result_version;
}
/**
* @task sync
*/
public function synchronizeWorkingCopyBeforeWrite() {
if (!$this->shouldEnableSynchronization()) {
return;
}
$repository = $this->getRepository();
$viewer = $this->getViewer();
$repository_phid = $repository->getPHID();
$device = AlmanacKeys::getLiveDevice();
$device_phid = $device->getPHID();
$table = new PhabricatorRepositoryWorkingCopyVersion();
$locked_connection = $table->establishConnection('w');
$write_lock = PhabricatorRepositoryWorkingCopyVersion::getWriteLock(
$repository_phid);
$write_lock->useSpecificConnection($locked_connection);
$lock_wait = phutil_units('2 minutes in seconds');
$this->logLine(
pht(
'Waiting up to %s second(s) for a cluster write lock...',
new PhutilNumber($lock_wait)));
try {
$start = PhabricatorTime::getNow();
$write_lock->lock($lock_wait);
$waited = (PhabricatorTime::getNow() - $start);
if ($waited) {
$this->logLine(
pht(
'Acquired write lock after %s second(s).',
new PhutilNumber($waited)));
} else {
$this->logLine(
pht(
'Acquired write lock immediately.'));
}
} catch (Exception $ex) {
throw new PhutilProxyException(
pht(
'Failed to acquire write lock after waiting %s second(s). You '.
'may be able to retry later.',
new PhutilNumber($lock_wait)));
}
$versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions(
$repository_phid);
foreach ($versions as $version) {
if (!$version->getIsWriting()) {
continue;
}
throw new Exception(
pht(
'An previous write to this repository was interrupted; refusing '.
'new writes. This issue requires operator intervention to resolve, '.
'see "Write Interruptions" in the "Cluster: Repositories" in the '.
'documentation for instructions.'));
}
try {
$max_version = $this->synchronizeWorkingCopyBeforeRead();
} catch (Exception $ex) {
$write_lock->unlock();
throw $ex;
}
$pid = getmypid();
$hash = Filesystem::readRandomCharacters(12);
$this->clusterWriteOwner = "{$pid}.{$hash}";
PhabricatorRepositoryWorkingCopyVersion::willWrite(
$locked_connection,
$repository_phid,
$device_phid,
array(
'userPHID' => $viewer->getPHID(),
'epoch' => PhabricatorTime::getNow(),
'devicePHID' => $device_phid,
),
$this->clusterWriteOwner);
$this->clusterWriteVersion = $max_version;
$this->clusterWriteLock = $write_lock;
}
+ public function synchronizeWorkingCopyAfterDiscovery($new_version) {
+ if (!$this->shouldEnableSynchronization()) {
+ return;
+ }
+
+ $repository = $this->getRepository();
+ $repository_phid = $repository->getPHID();
+ if ($repository->isHosted()) {
+ return;
+ }
+
+ $viewer = $this->getViewer();
+
+ $device = AlmanacKeys::getLiveDevice();
+ $device_phid = $device->getPHID();
+
+ // NOTE: We are not holding a lock here because this method is only called
+ // from PhabricatorRepositoryDiscoveryEngine, which already holds a device
+ // lock. Even if we do race here and record an older version, the
+ // consequences are mild: we only do extra work to correct it later.
+
+ $versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions(
+ $repository_phid);
+ $versions = mpull($versions, null, 'getDevicePHID');
+
+ $this_version = idx($versions, $device_phid);
+ if ($this_version) {
+ $this_version = (int)$this_version->getRepositoryVersion();
+ } else {
+ $this_version = -1;
+ }
+
+ if ($new_version > $this_version) {
+ PhabricatorRepositoryWorkingCopyVersion::updateVersion(
+ $repository_phid,
+ $device_phid,
+ $new_version);
+ }
+ }
+
+
/**
* @task sync
*/
public function synchronizeWorkingCopyAfterWrite() {
if (!$this->shouldEnableSynchronization()) {
return;
}
if (!$this->clusterWriteLock) {
throw new Exception(
pht(
'Trying to synchronize after write, but not holding a write '.
'lock!'));
}
$repository = $this->getRepository();
$repository_phid = $repository->getPHID();
$device = AlmanacKeys::getLiveDevice();
$device_phid = $device->getPHID();
// It is possible that we've lost the global lock while receiving the push.
// For example, the master database may have been restarted between the
// time we acquired the global lock and now, when the push has finished.
// We wrote a durable lock while we were holding the the global lock,
// essentially upgrading our lock. We can still safely release this upgraded
// lock even if we're no longer holding the global lock.
// If we fail to release the lock, the repository will be frozen until
// an operator can figure out what happened, so we try pretty hard to
// reconnect to the database and release the lock.
$now = PhabricatorTime::getNow();
$duration = phutil_units('5 minutes in seconds');
$try_until = $now + $duration;
$did_release = false;
$already_failed = false;
while (PhabricatorTime::getNow() <= $try_until) {
try {
// NOTE: This means we're still bumping the version when pushes fail. We
// could select only un-rejected events instead to bump a little less
// often.
$new_log = id(new PhabricatorRepositoryPushEventQuery())
->setViewer(PhabricatorUser::getOmnipotentUser())
->withRepositoryPHIDs(array($repository_phid))
->setLimit(1)
->executeOne();
$old_version = $this->clusterWriteVersion;
if ($new_log) {
$new_version = $new_log->getID();
} else {
$new_version = $old_version;
}
PhabricatorRepositoryWorkingCopyVersion::didWrite(
$repository_phid,
$device_phid,
$this->clusterWriteVersion,
$new_version,
$this->clusterWriteOwner);
$did_release = true;
break;
} catch (AphrontConnectionQueryException $ex) {
$connection_exception = $ex;
} catch (AphrontConnectionLostQueryException $ex) {
$connection_exception = $ex;
}
if (!$already_failed) {
$already_failed = true;
$this->logLine(
pht('CRITICAL. Failed to release cluster write lock!'));
$this->logLine(
pht(
'The connection to the master database was lost while receiving '.
'the write.'));
$this->logLine(
pht(
'This process will spend %s more second(s) attempting to '.
'recover, then give up.',
new PhutilNumber($duration)));
}
sleep(1);
}
if ($did_release) {
if ($already_failed) {
$this->logLine(
pht('RECOVERED. Link to master database was restored.'));
}
$this->logLine(pht('Released cluster write lock.'));
} else {
throw new Exception(
pht(
'Failed to reconnect to master database and release held write '.
'lock ("%s") on device "%s" for repository "%s" after trying '.
'for %s seconds(s). This repository will be frozen.',
$this->clusterWriteOwner,
$device->getName(),
$this->getDisplayName(),
new PhutilNumber($duration)));
}
// We can continue even if we've lost this lock, everything is still
// consistent.
try {
$this->clusterWriteLock->unlock();
} catch (Exception $ex) {
// Ignore.
}
$this->clusterWriteLock = null;
$this->clusterWriteOwner = null;
}
/* -( Internals )---------------------------------------------------------- */
/**
* @task internal
*/
private function shouldEnableSynchronization() {
$repository = $this->getRepository();
$service_phid = $repository->getAlmanacServicePHID();
if (!$service_phid) {
return false;
}
// TODO: For now, this is only supported for Git.
if (!$repository->isGit()) {
return false;
}
- // TODO: It may eventually make sense to try to version and synchronize
- // observed repositories (so that daemons don't do reads against out-of
- // date hosts), but don't bother for now.
- if (!$repository->isHosted()) {
- return false;
- }
-
$device = AlmanacKeys::getLiveDevice();
if (!$device) {
return false;
}
return true;
}
+ /**
+ * @task internal
+ */
+ private function synchornizeWorkingCopyFromRemote() {
+ $repository = $this->getRepository();
+ $device = AlmanacKeys::getLiveDevice();
+
+ $local_path = $repository->getLocalPath();
+ $fetch_uri = $repository->getRemoteURIEnvelope();
+
+ if ($repository->isGit()) {
+ $this->requireWorkingCopy();
+
+ $argv = array(
+ 'fetch --prune -- %P %s',
+ $fetch_uri,
+ '+refs/*:refs/*',
+ );
+ } else {
+ throw new Exception(pht('Remote sync only supported for git!'));
+ }
+
+ $future = DiffusionCommandEngine::newCommandEngine($repository)
+ ->setArgv($argv)
+ ->setSudoAsDaemon(true)
+ ->setCredentialPHID($repository->getCredentialPHID())
+ ->setProtocol($repository->getRemoteProtocol())
+ ->newFuture();
+
+ $future->setCWD($local_path);
+
+ try {
+ $future->resolvex();
+ } catch (Exception $ex) {
+ $this->logLine(
+ pht(
+ 'Synchronization of "%s" from remote failed: %s',
+ $device->getName(),
+ $ex->getMessage()));
+ throw $ex;
+ }
+ }
+
+
/**
* @task internal
*/
private function synchronizeWorkingCopyFromDevices(array $device_phids) {
$repository = $this->getRepository();
$service = $repository->loadAlmanacService();
if (!$service) {
throw new Exception(pht('Failed to load repository cluster service.'));
}
$device_map = array_fuse($device_phids);
$bindings = $service->getActiveBindings();
$fetchable = array();
foreach ($bindings as $binding) {
// We can't fetch from nodes which don't have the newest version.
$device_phid = $binding->getDevicePHID();
if (empty($device_map[$device_phid])) {
continue;
}
// TODO: For now, only fetch over SSH. We could support fetching over
// HTTP eventually.
if ($binding->getAlmanacPropertyValue('protocol') != 'ssh') {
continue;
}
$fetchable[] = $binding;
}
if (!$fetchable) {
throw new Exception(
pht(
'Leader lost: no up-to-date nodes in repository cluster are '.
'fetchable.'));
}
$caught = null;
foreach ($fetchable as $binding) {
try {
$this->synchronizeWorkingCopyFromBinding($binding);
$caught = null;
break;
} catch (Exception $ex) {
$caught = $ex;
}
}
if ($caught) {
throw $caught;
}
}
/**
* @task internal
*/
private function synchronizeWorkingCopyFromBinding($binding) {
$repository = $this->getRepository();
$device = AlmanacKeys::getLiveDevice();
$this->logLine(
pht(
'Synchronizing this device ("%s") from cluster leader ("%s") before '.
'read.',
$device->getName(),
$binding->getDevice()->getName()));
$fetch_uri = $repository->getClusterRepositoryURIFromBinding($binding);
$local_path = $repository->getLocalPath();
if ($repository->isGit()) {
- if (!Filesystem::pathExists($local_path)) {
- throw new Exception(
- pht(
- 'Repository "%s" does not have a working copy on this device '.
- 'yet, so it can not be synchronized. Wait for the daemons to '.
- 'construct one or run `bin/repository update %s` on this host '.
- '("%s") to build it explicitly.',
- $repository->getDisplayName(),
- $repository->getMonogram(),
- $device->getName()));
- }
+ $this->requireWorkingCopy();
$argv = array(
'fetch --prune -- %s %s',
$fetch_uri,
'+refs/*:refs/*',
);
} else {
throw new Exception(pht('Binding sync only supported for git!'));
}
$future = DiffusionCommandEngine::newCommandEngine($repository)
->setArgv($argv)
->setConnectAsDevice(true)
->setSudoAsDaemon(true)
->setProtocol($fetch_uri->getProtocol())
->newFuture();
$future->setCWD($local_path);
try {
$future->resolvex();
} catch (Exception $ex) {
$this->logLine(
pht(
'Synchronization of "%s" from leader "%s" failed: %s',
$device->getName(),
$binding->getDevice()->getName(),
$ex->getMessage()));
throw $ex;
}
}
/**
* @task internal
*/
private function logLine($message) {
return $this->logText("# {$message}\n");
}
/**
* @task internal
*/
private function logText($message) {
$log = $this->logger;
if ($log) {
$log->writeClusterEngineLogMessage($message);
}
return $this;
}
+
+ private function requireWorkingCopy() {
+ $repository = $this->getRepository();
+ $local_path = $repository->getLocalPath();
+
+ if (!Filesystem::pathExists($local_path)) {
+ $device = AlmanacKeys::getLiveDevice();
+
+ throw new Exception(
+ pht(
+ 'Repository "%s" does not have a working copy on this device '.
+ 'yet, so it can not be synchronized. Wait for the daemons to '.
+ 'construct one or run `bin/repository update %s` on this host '.
+ '("%s") to build it explicitly.',
+ $repository->getDisplayName(),
+ $repository->getMonogram(),
+ $device->getName()));
+ }
+ }
+
}
diff --git a/src/applications/diffusion/query/lowlevel/DiffusionLowLevelGitRefQuery.php b/src/applications/diffusion/query/lowlevel/DiffusionLowLevelGitRefQuery.php
index 1f17f5fe9f..038d833670 100644
--- a/src/applications/diffusion/query/lowlevel/DiffusionLowLevelGitRefQuery.php
+++ b/src/applications/diffusion/query/lowlevel/DiffusionLowLevelGitRefQuery.php
@@ -1,152 +1,155 @@
<?php
/**
* Execute and parse a low-level Git ref query using `git for-each-ref`. This
* is useful for returning a list of tags or branches.
*/
final class DiffusionLowLevelGitRefQuery extends DiffusionLowLevelQuery {
- private $isTag;
- private $isOriginBranch;
+ private $refTypes;
- public function withIsTag($is_tag) {
- $this->isTag = $is_tag;
- return $this;
- }
-
- public function withIsOriginBranch($is_origin_branch) {
- $this->isOriginBranch = $is_origin_branch;
+ public function withRefTypes(array $ref_types) {
+ $this->refTypes = $ref_types;
return $this;
}
protected function executeQuery() {
+ $ref_types = $this->refTypes;
+ if ($ref_types) {
+ $type_branch = PhabricatorRepositoryRefCursor::TYPE_BRANCH;
+ $type_tag = PhabricatorRepositoryRefCursor::TYPE_TAG;
+
+ $ref_types = array_fuse($ref_types);
+
+ $with_branches = isset($ref_types[$type_branch]);
+ $with_tags = isset($ref_types[$type_tag]);
+ } else {
+ $with_branches = true;
+ $with_tags = true;
+ }
+
$repository = $this->getRepository();
$prefixes = array();
- $any = ($this->isTag || $this->isOriginBranch);
- if (!$any) {
- throw new Exception(pht('Specify types of refs to query.'));
- }
-
- if ($this->isOriginBranch) {
+ if ($with_branches) {
if ($repository->isWorkingCopyBare()) {
$prefix = 'refs/heads/';
} else {
$remote = DiffusionGitBranch::DEFAULT_GIT_REMOTE;
$prefix = 'refs/remotes/'.$remote.'/';
}
$prefixes[] = $prefix;
}
- if ($this->isTag) {
+ if ($with_tags) {
$prefixes[] = 'refs/tags/';
}
$order = '-creatordate';
$futures = array();
foreach ($prefixes as $prefix) {
$futures[$prefix] = $repository->getLocalCommandFuture(
'for-each-ref --sort=%s --format=%s %s',
$order,
$this->getFormatString(),
$prefix);
}
// Resolve all the futures first. We want to iterate over them in prefix
// order, not resolution order.
foreach (new FutureIterator($futures) as $prefix => $future) {
$future->resolvex();
}
$results = array();
foreach ($futures as $prefix => $future) {
list($stdout) = $future->resolvex();
$stdout = rtrim($stdout);
if (!strlen($stdout)) {
continue;
}
// NOTE: Although git supports --count, we can't apply any offset or
// limit logic until the very end because we may encounter a HEAD which
// we want to discard.
$lines = explode("\n", $stdout);
foreach ($lines as $line) {
$fields = $this->extractFields($line);
$creator = $fields['creator'];
$matches = null;
if (preg_match('/^(.*) ([0-9]+) ([0-9+-]+)$/', $creator, $matches)) {
$fields['author'] = $matches[1];
$fields['epoch'] = (int)$matches[2];
} else {
$fields['author'] = null;
$fields['epoch'] = null;
}
$commit = nonempty($fields['*objectname'], $fields['objectname']);
$short = substr($fields['refname'], strlen($prefix));
if ($short == 'HEAD') {
continue;
}
$ref = id(new DiffusionRepositoryRef())
->setShortName($short)
->setCommitIdentifier($commit)
->setRawFields($fields);
$results[] = $ref;
}
}
return $results;
}
/**
* List of git `--format` fields we want to grab.
*/
private function getFields() {
return array(
'objectname',
'objecttype',
'refname',
'*objectname',
'*objecttype',
'subject',
'creator',
);
}
/**
* Get a string for `--format` which enumerates all the fields we want.
*/
private function getFormatString() {
$fields = $this->getFields();
foreach ($fields as $key => $field) {
$fields[$key] = '%('.$field.')';
}
return implode('%01', $fields);
}
/**
* Parse a line back into fields.
*/
private function extractFields($line) {
$fields = $this->getFields();
$parts = explode("\1", $line, count($fields));
$dict = array();
foreach ($fields as $index => $field) {
$dict[$field] = idx($parts, $index);
}
return $dict;
}
}
diff --git a/src/applications/diffusion/query/lowlevel/DiffusionLowLevelResolveRefsQuery.php b/src/applications/diffusion/query/lowlevel/DiffusionLowLevelResolveRefsQuery.php
index 4e9ed246d9..8f9493a67a 100644
--- a/src/applications/diffusion/query/lowlevel/DiffusionLowLevelResolveRefsQuery.php
+++ b/src/applications/diffusion/query/lowlevel/DiffusionLowLevelResolveRefsQuery.php
@@ -1,303 +1,306 @@
<?php
/**
* Resolves references (like short commit names, branch names, tag names, etc.)
* into canonical, stable commit identifiers. This query works for all
* repository types.
*
* This query will always resolve refs which can be resolved, but may need to
* perform VCS operations. A faster (but less complete) counterpart query is
* available in @{class:DiffusionCachedResolveRefsQuery}; that query can
* resolve most refs without VCS operations.
*/
final class DiffusionLowLevelResolveRefsQuery
extends DiffusionLowLevelQuery {
private $refs;
private $types;
public function withRefs(array $refs) {
$this->refs = $refs;
return $this;
}
public function withTypes(array $types) {
$this->types = $types;
return $this;
}
protected function executeQuery() {
if (!$this->refs) {
return array();
}
$repository = $this->getRepository();
if (!$repository->hasLocalWorkingCopy()) {
return array();
}
switch ($this->getRepository()->getVersionControlSystem()) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
$result = $this->resolveGitRefs();
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
$result = $this->resolveMercurialRefs();
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
$result = $this->resolveSubversionRefs();
break;
default:
throw new Exception(pht('Unsupported repository type!'));
}
if ($this->types !== null) {
$result = $this->filterRefsByType($result, $this->types);
}
return $result;
}
private function resolveGitRefs() {
$repository = $this->getRepository();
$unresolved = array_fuse($this->refs);
$results = array();
// First, resolve branches and tags.
$ref_map = id(new DiffusionLowLevelGitRefQuery())
->setRepository($repository)
- ->withIsTag(true)
- ->withIsOriginBranch(true)
+ ->withRefTypes(
+ array(
+ PhabricatorRepositoryRefCursor::TYPE_BRANCH,
+ PhabricatorRepositoryRefCursor::TYPE_TAG,
+ ))
->execute();
$ref_map = mgroup($ref_map, 'getShortName');
$tag_prefix = 'refs/tags/';
foreach ($unresolved as $ref) {
if (empty($ref_map[$ref])) {
continue;
}
foreach ($ref_map[$ref] as $result) {
$fields = $result->getRawFields();
$objectname = idx($fields, 'refname');
if (!strncmp($objectname, $tag_prefix, strlen($tag_prefix))) {
$type = 'tag';
} else {
$type = 'branch';
}
$info = array(
'type' => $type,
'identifier' => $result->getCommitIdentifier(),
);
if ($type == 'tag') {
$alternate = idx($fields, 'objectname');
if ($alternate) {
$info['alternate'] = $alternate;
}
}
$results[$ref][] = $info;
}
unset($unresolved[$ref]);
}
// If we resolved everything, we're done.
if (!$unresolved) {
return $results;
}
// Try to resolve anything else. This stuff either doesn't exist or is
// some ref like "HEAD^^^".
$future = $repository->getLocalCommandFuture('cat-file --batch-check');
$future->write(implode("\n", $unresolved));
list($stdout) = $future->resolvex();
$lines = explode("\n", rtrim($stdout, "\n"));
if (count($lines) !== count($unresolved)) {
throw new Exception(
pht(
'Unexpected line count from `%s`!',
'git cat-file'));
}
$hits = array();
$tags = array();
$lines = array_combine($unresolved, $lines);
foreach ($lines as $ref => $line) {
$parts = explode(' ', $line);
if (count($parts) < 2) {
throw new Exception(
pht(
'Failed to parse `%s` output: %s',
'git cat-file',
$line));
}
list($identifier, $type) = $parts;
if ($type == 'missing') {
// This is either an ambiguous reference which resolves to several
// objects, or an invalid reference. For now, always treat it as
// invalid. It would be nice to resolve all possibilities for
// ambiguous references at some point, although the strategy for doing
// so isn't clear to me.
continue;
}
switch ($type) {
case 'commit':
break;
case 'tag':
$tags[] = $identifier;
break;
default:
throw new Exception(
pht(
'Unexpected object type from `%s`: %s',
'git cat-file',
$line));
}
$hits[] = array(
'ref' => $ref,
'type' => $type,
'identifier' => $identifier,
);
}
$tag_map = array();
if ($tags) {
// If some of the refs were tags, just load every tag in order to figure
// out which commits they map to. This might be somewhat inefficient in
// repositories with a huge number of tags.
$tag_refs = id(new DiffusionLowLevelGitRefQuery())
->setRepository($repository)
->withIsTag(true)
->executeQuery();
foreach ($tag_refs as $tag_ref) {
$tag_map[$tag_ref->getShortName()] = $tag_ref->getCommitIdentifier();
}
}
$results = array();
foreach ($hits as $hit) {
$type = $hit['type'];
$ref = $hit['ref'];
$alternate = null;
if ($type == 'tag') {
$alternate = $identifier;
$identifier = idx($tag_map, $ref);
if (!$identifier) {
throw new Exception(
pht(
"Failed to look up tag '%s'!",
$ref));
}
}
$result = array(
'type' => $type,
'identifier' => $identifier,
);
if ($alternate !== null) {
$result['alternate'] = $alternate;
}
$results[$ref][] = $result;
}
return $results;
}
private function resolveMercurialRefs() {
$repository = $this->getRepository();
// First, pull all of the branch heads in the repository. Doing this in
// bulk is much faster than querying each individual head if we're
// checking even a small number of refs.
$branches = id(new DiffusionLowLevelMercurialBranchesQuery())
->setRepository($repository)
->executeQuery();
$branches = mgroup($branches, 'getShortName');
$results = array();
$unresolved = $this->refs;
foreach ($unresolved as $key => $ref) {
if (empty($branches[$ref])) {
continue;
}
foreach ($branches[$ref] as $branch) {
$fields = $branch->getRawFields();
$results[$ref][] = array(
'type' => 'branch',
'identifier' => $branch->getCommitIdentifier(),
'closed' => idx($fields, 'closed', false),
);
}
unset($unresolved[$key]);
}
if (!$unresolved) {
return $results;
}
// If we still have unresolved refs (which might be things like "tip"),
// try to resolve them individually.
$futures = array();
foreach ($unresolved as $ref) {
$futures[$ref] = $repository->getLocalCommandFuture(
'log --template=%s --rev %s',
'{node}',
hgsprintf('%s', $ref));
}
foreach (new FutureIterator($futures) as $ref => $future) {
try {
list($stdout) = $future->resolvex();
} catch (CommandException $ex) {
if (preg_match('/ambiguous identifier/', $ex->getStdErr())) {
// This indicates that the ref ambiguously matched several things.
// Eventually, it would be nice to return all of them, but it is
// unclear how to best do that. For now, treat it as a miss instead.
continue;
}
if (preg_match('/unknown revision/', $ex->getStdErr())) {
// No matches for this ref.
continue;
}
throw $ex;
}
// It doesn't look like we can figure out the type (commit/branch/rev)
// from this output very easily. For now, just call everything a commit.
$type = 'commit';
$results[$ref][] = array(
'type' => $type,
'identifier' => trim($stdout),
);
}
return $results;
}
private function resolveSubversionRefs() {
// We don't have any VCS logic for Subversion, so just use the cached
// query.
return id(new DiffusionCachedResolveRefsQuery())
->setRepository($this->getRepository())
->withRefs($this->refs)
->execute();
}
}
diff --git a/src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php b/src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
index 78a40986e3..9e0e13c720 100644
--- a/src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
+++ b/src/applications/repository/engine/PhabricatorRepositoryDiscoveryEngine.php
@@ -1,645 +1,705 @@
<?php
/**
* @task discover Discovering Repositories
* @task svn Discovering Subversion Repositories
* @task git Discovering Git Repositories
* @task hg Discovering Mercurial Repositories
* @task internal Internals
*/
final class PhabricatorRepositoryDiscoveryEngine
extends PhabricatorRepositoryEngine {
private $repairMode;
private $commitCache = array();
private $workingSet = array();
const MAX_COMMIT_CACHE_SIZE = 2048;
/* -( Discovering Repositories )------------------------------------------- */
public function setRepairMode($repair_mode) {
$this->repairMode = $repair_mode;
return $this;
}
public function getRepairMode() {
return $this->repairMode;
}
/**
* @task discovery
*/
public function discoverCommits() {
$repository = $this->getRepository();
$lock = $this->newRepositoryLock($repository, 'repo.look', false);
try {
$lock->lock();
} catch (PhutilLockException $ex) {
throw new DiffusionDaemonLockException(
pht(
'Another process is currently discovering repository "%s", '.
'skipping discovery.',
$repository->getDisplayName()));
}
try {
$result = $this->discoverCommitsWithLock();
} catch (Exception $ex) {
$lock->unlock();
throw $ex;
}
$lock->unlock();
return $result;
}
private function discoverCommitsWithLock() {
$repository = $this->getRepository();
+ $viewer = $this->getViewer();
$vcs = $repository->getVersionControlSystem();
switch ($vcs) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
$refs = $this->discoverSubversionCommits();
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
$refs = $this->discoverMercurialCommits();
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
$refs = $this->discoverGitCommits();
break;
default:
throw new Exception(pht("Unknown VCS '%s'!", $vcs));
}
if ($this->isInitialImport($refs)) {
$this->log(
pht(
'Discovered more than %s commit(s) in an empty repository, '.
'marking repository as importing.',
new PhutilNumber(PhabricatorRepository::IMPORT_THRESHOLD)));
$repository->markImporting();
}
// Clear the working set cache.
$this->workingSet = array();
// Record discovered commits and mark them in the cache.
foreach ($refs as $ref) {
$this->recordCommit(
$repository,
$ref->getIdentifier(),
$ref->getEpoch(),
$ref->getCanCloseImmediately(),
$ref->getParents());
$this->commitCache[$ref->getIdentifier()] = true;
}
+ $version = $this->getObservedVersion($repository);
+ if ($version !== null) {
+ id(new DiffusionRepositoryClusterEngine())
+ ->setViewer($viewer)
+ ->setRepository($repository)
+ ->synchronizeWorkingCopyAfterDiscovery($version);
+ }
+
return $refs;
}
/* -( Discovering Git Repositories )--------------------------------------- */
/**
* @task git
*/
private function discoverGitCommits() {
$repository = $this->getRepository();
if (!$repository->isHosted()) {
$this->verifyGitOrigin($repository);
}
+ // TODO: This should also import tags, but some of the logic is still
+ // branch-specific today.
+
$branches = id(new DiffusionLowLevelGitRefQuery())
->setRepository($repository)
- ->withIsOriginBranch(true)
+ ->withRefTypes(
+ array(
+ PhabricatorRepositoryRefCursor::TYPE_BRANCH,
+ ))
->execute();
if (!$branches) {
// This repository has no branches at all, so we don't need to do
// anything. Generally, this means the repository is empty.
return array();
}
$branches = $this->sortBranches($branches);
$branches = mpull($branches, 'getCommitIdentifier', 'getShortName');
$this->log(
pht(
'Discovering commits in repository "%s".',
$repository->getDisplayName()));
$this->fillCommitCache(array_values($branches));
$refs = array();
foreach ($branches as $name => $commit) {
$this->log(pht('Examining branch "%s", at "%s".', $name, $commit));
if (!$repository->shouldTrackBranch($name)) {
$this->log(pht('Skipping, branch is untracked.'));
continue;
}
if ($this->isKnownCommit($commit)) {
$this->log(pht('Skipping, HEAD is known.'));
continue;
}
$this->log(pht('Looking for new commits.'));
$branch_refs = $this->discoverStreamAncestry(
new PhabricatorGitGraphStream($repository, $commit),
$commit,
$repository->shouldAutocloseBranch($name));
$this->didDiscoverRefs($branch_refs);
$refs[] = $branch_refs;
}
return array_mergev($refs);
}
/* -( Discovering Subversion Repositories )-------------------------------- */
/**
* @task svn
*/
private function discoverSubversionCommits() {
$repository = $this->getRepository();
if (!$repository->isHosted()) {
$this->verifySubversionRoot($repository);
}
$upper_bound = null;
$limit = 1;
$refs = array();
do {
// Find all the unknown commits on this path. Note that we permit
// importing an SVN subdirectory rather than the entire repository, so
// commits may be nonsequential.
if ($upper_bound === null) {
$at_rev = 'HEAD';
} else {
$at_rev = ($upper_bound - 1);
}
try {
list($xml, $stderr) = $repository->execxRemoteCommand(
'log --xml --quiet --limit %d %s',
$limit,
$repository->getSubversionBaseURI($at_rev));
} catch (CommandException $ex) {
$stderr = $ex->getStdErr();
if (preg_match('/(path|File) not found/', $stderr)) {
// We've gone all the way back through history and this path was not
// affected by earlier commits.
break;
}
throw $ex;
}
$xml = phutil_utf8ize($xml);
$log = new SimpleXMLElement($xml);
foreach ($log->logentry as $entry) {
$identifier = (int)$entry['revision'];
$epoch = (int)strtotime((string)$entry->date[0]);
$refs[$identifier] = id(new PhabricatorRepositoryCommitRef())
->setIdentifier($identifier)
->setEpoch($epoch)
->setCanCloseImmediately(true);
if ($upper_bound === null) {
$upper_bound = $identifier;
} else {
$upper_bound = min($upper_bound, $identifier);
}
}
// Discover 2, 4, 8, ... 256 logs at a time. This allows us to initially
// import large repositories fairly quickly, while pulling only as much
// data as we need in the common case (when we've already imported the
// repository and are just grabbing one commit at a time).
$limit = min($limit * 2, 256);
} while ($upper_bound > 1 && !$this->isKnownCommit($upper_bound));
krsort($refs);
while ($refs && $this->isKnownCommit(last($refs)->getIdentifier())) {
array_pop($refs);
}
$refs = array_reverse($refs);
$this->didDiscoverRefs($refs);
return $refs;
}
private function verifySubversionRoot(PhabricatorRepository $repository) {
list($xml) = $repository->execxRemoteCommand(
'info --xml %s',
$repository->getSubversionPathURI());
$xml = phutil_utf8ize($xml);
$xml = new SimpleXMLElement($xml);
$remote_root = (string)($xml->entry[0]->repository[0]->root[0]);
$expect_root = $repository->getSubversionPathURI();
$normal_type_svn = PhabricatorRepositoryURINormalizer::TYPE_SVN;
$remote_normal = id(new PhabricatorRepositoryURINormalizer(
$normal_type_svn,
$remote_root))->getNormalizedPath();
$expect_normal = id(new PhabricatorRepositoryURINormalizer(
$normal_type_svn,
$expect_root))->getNormalizedPath();
if ($remote_normal != $expect_normal) {
throw new Exception(
pht(
'Repository "%s" does not have a correctly configured remote URI. '.
'The remote URI for a Subversion repository MUST point at the '.
'repository root. The root for this repository is "%s", but the '.
'configured URI is "%s". To resolve this error, set the remote URI '.
'to point at the repository root. If you want to import only part '.
'of a Subversion repository, use the "Import Only" option.',
$repository->getDisplayName(),
$remote_root,
$expect_root));
}
}
/* -( Discovering Mercurial Repositories )--------------------------------- */
/**
* @task hg
*/
private function discoverMercurialCommits() {
$repository = $this->getRepository();
$branches = id(new DiffusionLowLevelMercurialBranchesQuery())
->setRepository($repository)
->execute();
$this->fillCommitCache(mpull($branches, 'getCommitIdentifier'));
$refs = array();
foreach ($branches as $branch) {
// NOTE: Mercurial branches may have multiple heads, so the names may
// not be unique.
$name = $branch->getShortName();
$commit = $branch->getCommitIdentifier();
$this->log(pht('Examining branch "%s" head "%s".', $name, $commit));
if (!$repository->shouldTrackBranch($name)) {
$this->log(pht('Skipping, branch is untracked.'));
continue;
}
if ($this->isKnownCommit($commit)) {
$this->log(pht('Skipping, this head is a known commit.'));
continue;
}
$this->log(pht('Looking for new commits.'));
$branch_refs = $this->discoverStreamAncestry(
new PhabricatorMercurialGraphStream($repository, $commit),
$commit,
$close_immediately = true);
$this->didDiscoverRefs($branch_refs);
$refs[] = $branch_refs;
}
return array_mergev($refs);
}
/* -( Internals )---------------------------------------------------------- */
private function discoverStreamAncestry(
PhabricatorRepositoryGraphStream $stream,
$commit,
$close_immediately) {
$discover = array($commit);
$graph = array();
$seen = array();
// Find all the reachable, undiscovered commits. Build a graph of the
// edges.
while ($discover) {
$target = array_pop($discover);
if (empty($graph[$target])) {
$graph[$target] = array();
}
$parents = $stream->getParents($target);
foreach ($parents as $parent) {
if ($this->isKnownCommit($parent)) {
continue;
}
$graph[$target][$parent] = true;
if (empty($seen[$parent])) {
$seen[$parent] = true;
$discover[] = $parent;
}
}
}
// Now, sort them topographically.
$commits = $this->reduceGraph($graph);
$refs = array();
foreach ($commits as $commit) {
$refs[] = id(new PhabricatorRepositoryCommitRef())
->setIdentifier($commit)
->setEpoch($stream->getCommitDate($commit))
->setCanCloseImmediately($close_immediately)
->setParents($stream->getParents($commit));
}
return $refs;
}
private function reduceGraph(array $edges) {
foreach ($edges as $commit => $parents) {
$edges[$commit] = array_keys($parents);
}
$graph = new PhutilDirectedScalarGraph();
$graph->addNodes($edges);
$commits = $graph->getTopographicallySortedNodes();
// NOTE: We want the most ancestral nodes first, so we need to reverse the
// list we get out of AbstractDirectedGraph.
$commits = array_reverse($commits);
return $commits;
}
private function isKnownCommit($identifier) {
if (isset($this->commitCache[$identifier])) {
return true;
}
if (isset($this->workingSet[$identifier])) {
return true;
}
if ($this->repairMode) {
// In repair mode, rediscover the entire repository, ignoring the
// database state. We can hit the local cache above, but if we miss it
// stop the script from going to the database cache.
return false;
}
$this->fillCommitCache(array($identifier));
return isset($this->commitCache[$identifier]);
}
private function fillCommitCache(array $identifiers) {
if (!$identifiers) {
return;
}
$commits = id(new PhabricatorRepositoryCommit())->loadAllWhere(
'repositoryID = %d AND commitIdentifier IN (%Ls)',
$this->getRepository()->getID(),
$identifiers);
foreach ($commits as $commit) {
$this->commitCache[$commit->getCommitIdentifier()] = true;
}
while (count($this->commitCache) > self::MAX_COMMIT_CACHE_SIZE) {
array_shift($this->commitCache);
}
}
/**
* Sort branches so we process closeable branches first. This makes the
* whole import process a little cheaper, since we can close these commits
* the first time through rather than catching them in the refs step.
*
* @task internal
*
* @param list<DiffusionRepositoryRef> List of branch heads.
* @return list<DiffusionRepositoryRef> Sorted list of branch heads.
*/
private function sortBranches(array $branches) {
$repository = $this->getRepository();
$head_branches = array();
$tail_branches = array();
foreach ($branches as $branch) {
$name = $branch->getShortName();
if ($repository->shouldAutocloseBranch($name)) {
$head_branches[] = $branch;
} else {
$tail_branches[] = $branch;
}
}
return array_merge($head_branches, $tail_branches);
}
private function recordCommit(
PhabricatorRepository $repository,
$commit_identifier,
$epoch,
$close_immediately,
array $parents) {
$commit = new PhabricatorRepositoryCommit();
$commit->setRepositoryID($repository->getID());
$commit->setCommitIdentifier($commit_identifier);
$commit->setEpoch($epoch);
if ($close_immediately) {
$commit->setImportStatus(PhabricatorRepositoryCommit::IMPORTED_CLOSEABLE);
}
$data = new PhabricatorRepositoryCommitData();
$conn_w = $repository->establishConnection('w');
try {
// If this commit has parents, look up their IDs. The parent commits
// should always exist already.
$parent_ids = array();
if ($parents) {
$parent_rows = queryfx_all(
$conn_w,
'SELECT id, commitIdentifier FROM %T
WHERE commitIdentifier IN (%Ls) AND repositoryID = %d',
$commit->getTableName(),
$parents,
$repository->getID());
$parent_map = ipull($parent_rows, 'id', 'commitIdentifier');
foreach ($parents as $parent) {
if (empty($parent_map[$parent])) {
throw new Exception(
pht('Unable to identify parent "%s"!', $parent));
}
$parent_ids[] = $parent_map[$parent];
}
} else {
// Write an explicit 0 so we can distinguish between "really no
// parents" and "data not available".
if (!$repository->isSVN()) {
$parent_ids = array(0);
}
}
$commit->openTransaction();
$commit->save();
$data->setCommitID($commit->getID());
$data->save();
foreach ($parent_ids as $parent_id) {
queryfx(
$conn_w,
'INSERT IGNORE INTO %T (childCommitID, parentCommitID)
VALUES (%d, %d)',
PhabricatorRepository::TABLE_PARENTS,
$commit->getID(),
$parent_id);
}
$commit->saveTransaction();
$this->insertTask($repository, $commit);
queryfx(
$conn_w,
'INSERT INTO %T (repositoryID, size, lastCommitID, epoch)
VALUES (%d, 1, %d, %d)
ON DUPLICATE KEY UPDATE
size = size + 1,
lastCommitID =
IF(VALUES(epoch) > epoch, VALUES(lastCommitID), lastCommitID),
epoch = IF(VALUES(epoch) > epoch, VALUES(epoch), epoch)',
PhabricatorRepository::TABLE_SUMMARY,
$repository->getID(),
$commit->getID(),
$epoch);
if ($this->repairMode) {
// Normally, the query should throw a duplicate key exception. If we
// reach this in repair mode, we've actually performed a repair.
$this->log(pht('Repaired commit "%s".', $commit_identifier));
}
PhutilEventEngine::dispatchEvent(
new PhabricatorEvent(
PhabricatorEventType::TYPE_DIFFUSION_DIDDISCOVERCOMMIT,
array(
'repository' => $repository,
'commit' => $commit,
)));
} catch (AphrontDuplicateKeyQueryException $ex) {
$commit->killTransaction();
// Ignore. This can happen because we discover the same new commit
// more than once when looking at history, or because of races or
// data inconsistency or cosmic radiation; in any case, we're still
// in a good state if we ignore the failure.
}
}
private function didDiscoverRefs(array $refs) {
foreach ($refs as $ref) {
$this->workingSet[$ref->getIdentifier()] = true;
}
}
private function insertTask(
PhabricatorRepository $repository,
PhabricatorRepositoryCommit $commit,
$data = array()) {
$vcs = $repository->getVersionControlSystem();
switch ($vcs) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
$class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
$class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
$class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker';
break;
default:
throw new Exception(pht("Unknown repository type '%s'!", $vcs));
}
$data['commitID'] = $commit->getID();
PhabricatorWorker::scheduleTask($class, $data);
}
private function isInitialImport(array $refs) {
$commit_count = count($refs);
if ($commit_count <= PhabricatorRepository::IMPORT_THRESHOLD) {
// If we fetched a small number of commits, assume it's an initial
// commit or a stack of a few initial commits.
return false;
}
$viewer = $this->getViewer();
$repository = $this->getRepository();
$any_commits = id(new DiffusionCommitQuery())
->setViewer($viewer)
->withRepository($repository)
->setLimit(1)
->execute();
if ($any_commits) {
// If the repository already has commits, this isn't an import.
return false;
}
return true;
}
+
+ private function getObservedVersion(PhabricatorRepository $repository) {
+ if ($repository->isHosted()) {
+ return null;
+ }
+
+ if ($repository->isGit()) {
+ return $this->getGitObservedVersion($repository);
+ }
+
+ return null;
+ }
+
+ private function getGitObservedVersion(PhabricatorRepository $repository) {
+ $refs = id(new DiffusionLowLevelGitRefQuery())
+ ->setRepository($repository)
+ ->execute();
+ if (!$refs) {
+ return null;
+ }
+
+ // In Git, the observed version is the most recently discovered commit
+ // at any repository HEAD. It's possible for this to regress temporarily
+ // if a branch is pushed and then deleted. This is acceptable because it
+ // doesn't do anything meaningfully bad and will fix itself on the next
+ // push.
+
+ $ref_identifiers = mpull($refs, 'getCommitIdentifier');
+ $ref_identifiers = array_fuse($ref_identifiers);
+
+ $version = queryfx_one(
+ $repository->establishConnection('w'),
+ 'SELECT MAX(id) version FROM %T WHERE repositoryID = %d
+ AND commitIdentifier IN (%Ls)',
+ id(new PhabricatorRepositoryCommit())->getTableName(),
+ $repository->getID(),
+ $ref_identifiers);
+
+ if (!$version) {
+ return null;
+ }
+
+ return (int)$version['version'];
+ }
+
}
diff --git a/src/applications/repository/engine/PhabricatorRepositoryPullEngine.php b/src/applications/repository/engine/PhabricatorRepositoryPullEngine.php
index 9e151ecd81..3d7dce90fa 100644
--- a/src/applications/repository/engine/PhabricatorRepositoryPullEngine.php
+++ b/src/applications/repository/engine/PhabricatorRepositoryPullEngine.php
@@ -1,593 +1,593 @@
<?php
/**
* Manages execution of `git pull` and `hg pull` commands for
* @{class:PhabricatorRepository} objects. Used by
* @{class:PhabricatorRepositoryPullLocalDaemon}.
*
* This class also covers initial working copy setup through `git clone`,
* `git init`, `hg clone`, `hg init`, or `svnadmin create`.
*
* @task pull Pulling Working Copies
* @task git Pulling Git Working Copies
* @task hg Pulling Mercurial Working Copies
* @task svn Pulling Subversion Working Copies
* @task internal Internals
*/
final class PhabricatorRepositoryPullEngine
extends PhabricatorRepositoryEngine {
/* -( Pulling Working Copies )--------------------------------------------- */
public function pullRepository() {
$repository = $this->getRepository();
$lock = $this->newRepositoryLock($repository, 'repo.pull', true);
try {
$lock->lock();
} catch (PhutilLockException $ex) {
throw new DiffusionDaemonLockException(
pht(
'Another process is currently updating repository "%s", '.
'skipping pull.',
$repository->getDisplayName()));
}
try {
$result = $this->pullRepositoryWithLock();
} catch (Exception $ex) {
$lock->unlock();
throw $ex;
}
$lock->unlock();
return $result;
}
private function pullRepositoryWithLock() {
$repository = $this->getRepository();
$viewer = PhabricatorUser::getOmnipotentUser();
$is_hg = false;
$is_git = false;
$is_svn = false;
$vcs = $repository->getVersionControlSystem();
switch ($vcs) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
// We never pull a local copy of non-hosted Subversion repositories.
if (!$repository->isHosted()) {
$this->skipPull(
pht(
'Repository "%s" is a non-hosted Subversion repository, which '.
'does not require a local working copy to be pulled.',
$repository->getDisplayName()));
return;
}
$is_svn = true;
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
$is_git = true;
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
$is_hg = true;
break;
default:
$this->abortPull(pht('Unknown VCS "%s"!', $vcs));
break;
}
$local_path = $repository->getLocalPath();
if ($local_path === null) {
$this->abortPull(
pht(
'No local path is configured for repository "%s".',
$repository->getDisplayName()));
}
try {
$dirname = dirname($local_path);
if (!Filesystem::pathExists($dirname)) {
Filesystem::createDirectory($dirname, 0755, $recursive = true);
}
if (!Filesystem::pathExists($local_path)) {
$this->logPull(
pht(
'Creating a new working copy for repository "%s".',
$repository->getDisplayName()));
if ($is_git) {
$this->executeGitCreate();
} else if ($is_hg) {
$this->executeMercurialCreate();
} else {
$this->executeSubversionCreate();
}
- } else {
- if (!$repository->isHosted()) {
- $this->logPull(
- pht(
- 'Updating the working copy for repository "%s".',
- $repository->getDisplayName()));
- if ($is_git) {
- $this->verifyGitOrigin($repository);
- $this->executeGitUpdate();
- } else if ($is_hg) {
- $this->executeMercurialUpdate();
- }
+ }
+
+ id(new DiffusionRepositoryClusterEngine())
+ ->setViewer($viewer)
+ ->setRepository($repository)
+ ->synchronizeWorkingCopyBeforeRead();
+
+ if (!$repository->isHosted()) {
+ $this->logPull(
+ pht(
+ 'Updating the working copy for repository "%s".',
+ $repository->getDisplayName()));
+ if ($is_git) {
+ $this->verifyGitOrigin($repository);
+ $this->executeGitUpdate();
+ } else if ($is_hg) {
+ $this->executeMercurialUpdate();
}
}
if ($repository->isHosted()) {
- id(new DiffusionRepositoryClusterEngine())
- ->setViewer($viewer)
- ->setRepository($repository)
- ->synchronizeWorkingCopyBeforeRead();
-
if ($is_git) {
$this->installGitHook();
} else if ($is_svn) {
$this->installSubversionHook();
} else if ($is_hg) {
$this->installMercurialHook();
}
foreach ($repository->getHookDirectories() as $directory) {
$this->installHookDirectory($directory);
}
}
} catch (Exception $ex) {
$this->abortPull(
pht(
"Pull of '%s' failed: %s",
$repository->getDisplayName(),
$ex->getMessage()),
$ex);
}
$this->donePull();
return $this;
}
private function skipPull($message) {
$this->log('%s', $message);
$this->donePull();
}
private function abortPull($message, Exception $ex = null) {
$code_error = PhabricatorRepositoryStatusMessage::CODE_ERROR;
$this->updateRepositoryInitStatus($code_error, $message);
if ($ex) {
throw $ex;
} else {
throw new Exception($message);
}
}
private function logPull($message) {
$code_working = PhabricatorRepositoryStatusMessage::CODE_WORKING;
$this->updateRepositoryInitStatus($code_working, $message);
$this->log('%s', $message);
}
private function donePull() {
$code_okay = PhabricatorRepositoryStatusMessage::CODE_OKAY;
$this->updateRepositoryInitStatus($code_okay);
}
private function updateRepositoryInitStatus($code, $message = null) {
$this->getRepository()->writeStatusMessage(
PhabricatorRepositoryStatusMessage::TYPE_INIT,
$code,
array(
'message' => $message,
));
}
private function installHook($path) {
$this->log('%s', pht('Installing commit hook to "%s"...', $path));
$repository = $this->getRepository();
$identifier = $this->getHookContextIdentifier($repository);
$root = dirname(phutil_get_library_root('phabricator'));
$bin = $root.'/bin/commit-hook';
$full_php_path = Filesystem::resolveBinary('php');
$cmd = csprintf(
'exec %s -f %s -- %s "$@"',
$full_php_path,
$bin,
$identifier);
$hook = "#!/bin/sh\nexport TERM=dumb\n{$cmd}\n";
Filesystem::writeFile($path, $hook);
Filesystem::changePermissions($path, 0755);
}
private function installHookDirectory($path) {
$readme = pht(
"To add custom hook scripts to this repository, add them to this ".
"directory.\n\nPhabricator will run any executables in this directory ".
"after running its own checks, as though they were normal hook ".
"scripts.");
Filesystem::createDirectory($path, 0755);
Filesystem::writeFile($path.'/README', $readme);
}
private function getHookContextIdentifier(PhabricatorRepository $repository) {
$identifier = $repository->getPHID();
$instance = PhabricatorEnv::getEnvConfig('cluster.instance');
if (strlen($instance)) {
$identifier = "{$identifier}:{$instance}";
}
return $identifier;
}
/* -( Pulling Git Working Copies )----------------------------------------- */
/**
* @task git
*/
private function executeGitCreate() {
$repository = $this->getRepository();
$path = rtrim($repository->getLocalPath(), '/');
if ($repository->isHosted()) {
$repository->execxRemoteCommand(
'init --bare -- %s',
$path);
} else {
$repository->execxRemoteCommand(
'clone --bare -- %P %s',
$repository->getRemoteURIEnvelope(),
$path);
}
}
/**
* @task git
*/
private function executeGitUpdate() {
$repository = $this->getRepository();
list($err, $stdout) = $repository->execLocalCommand(
'rev-parse --show-toplevel');
$message = null;
$path = $repository->getLocalPath();
if ($err) {
// Try to raise a more tailored error message in the more common case
// of the user creating an empty directory. (We could try to remove it,
// but might not be able to, and it's much simpler to raise a good
// message than try to navigate those waters.)
if (is_dir($path)) {
$files = Filesystem::listDirectory($path, $include_hidden = true);
if (!$files) {
$message = pht(
"Expected to find a git repository at '%s', but there ".
"is an empty directory there. Remove the directory: the daemon ".
"will run '%s' for you.",
$path,
'git clone');
} else {
$message = pht(
"Expected to find a git repository at '%s', but there is ".
"a non-repository directory (with other stuff in it) there. Move ".
"or remove this directory (or reconfigure the repository to use a ".
"different directory), and then either clone a repository ".
"yourself or let the daemon do it.",
$path);
}
} else if (is_file($path)) {
$message = pht(
"Expected to find a git repository at '%s', but there is a ".
"file there instead. Remove it and let the daemon clone a ".
"repository for you.",
$path);
} else {
$message = pht(
"Expected to find a git repository at '%s', but did not.",
$path);
}
} else {
$repo_path = rtrim($stdout, "\n");
if (empty($repo_path)) {
// This can mean one of two things: we're in a bare repository, or
// we're inside a git repository inside another git repository. Since
// the first is dramatically more likely now that we perform bare
// clones and I don't have a great way to test for the latter, assume
// we're OK.
} else if (!Filesystem::pathsAreEquivalent($repo_path, $path)) {
$err = true;
$message = pht(
"Expected to find repo at '%s', but the actual git repository root ".
"for this directory is '%s'. Something is misconfigured. ".
"The repository's 'Local Path' should be set to some place where ".
"the daemon can check out a working copy, ".
"and should not be inside another git repository.",
$path,
$repo_path);
}
}
if ($err && $repository->canDestroyWorkingCopy()) {
phlog(
pht(
"Repository working copy at '%s' failed sanity check; ".
"destroying and re-cloning. %s",
$path,
$message));
Filesystem::remove($path);
$this->executeGitCreate();
} else if ($err) {
throw new Exception($message);
}
$retry = false;
do {
// This is a local command, but needs credentials.
if ($repository->isWorkingCopyBare()) {
// For bare working copies, we need this magic incantation.
$future = $repository->getRemoteCommandFuture(
'fetch origin %s --prune',
'+refs/heads/*:refs/heads/*');
} else {
$future = $repository->getRemoteCommandFuture(
'fetch --all --prune');
}
$future->setCWD($path);
list($err, $stdout, $stderr) = $future->resolve();
if ($err && !$retry && $repository->canDestroyWorkingCopy()) {
$retry = true;
// Fix remote origin url if it doesn't match our configuration
$origin_url = $repository->execLocalCommand(
'config --get remote.origin.url');
$remote_uri = $repository->getRemoteURIEnvelope();
if ($origin_url != $remote_uri->openEnvelope()) {
$repository->execLocalCommand(
'remote set-url origin %P',
$remote_uri);
}
} else if ($err) {
throw new CommandException(
pht('Failed to fetch changes!'),
$future->getCommand(),
$err,
$stdout,
$stderr);
} else {
$retry = false;
}
} while ($retry);
}
/**
* @task git
*/
private function installGitHook() {
$repository = $this->getRepository();
$root = $repository->getLocalPath();
if ($repository->isWorkingCopyBare()) {
$path = '/hooks/pre-receive';
} else {
$path = '/.git/hooks/pre-receive';
}
$this->installHook($root.$path);
}
/* -( Pulling Mercurial Working Copies )----------------------------------- */
/**
* @task hg
*/
private function executeMercurialCreate() {
$repository = $this->getRepository();
$path = rtrim($repository->getLocalPath(), '/');
if ($repository->isHosted()) {
$repository->execxRemoteCommand(
'init -- %s',
$path);
} else {
$remote = $repository->getRemoteURIEnvelope();
// NOTE: Mercurial prior to 3.2.4 has an severe command injection
// vulnerability. See: <http://bit.ly/19B58E9>
// On vulnerable versions of Mercurial, we refuse to clone remotes which
// contain characters which may be interpreted by the shell.
$hg_version = PhabricatorRepositoryVersion::getMercurialVersion();
$is_vulnerable = version_compare($hg_version, '3.2.4', '<');
if ($is_vulnerable) {
$cleartext = $remote->openEnvelope();
// The use of "%R" here is an attempt to limit collateral damage
// for normal URIs because it isn't clear how long this vulnerability
// has been around for.
$escaped = csprintf('%R', $cleartext);
if ((string)$escaped !== (string)$cleartext) {
throw new Exception(
pht(
'You have an old version of Mercurial (%s) which has a severe '.
'command injection security vulnerability. The remote URI for '.
'this repository (%s) is potentially unsafe. Upgrade Mercurial '.
'to at least 3.2.4 to clone it.',
$hg_version,
$repository->getMonogram()));
}
}
try {
$repository->execxRemoteCommand(
'clone --noupdate -- %P %s',
$remote,
$path);
} catch (Exception $ex) {
$message = $ex->getMessage();
$message = $this->censorMercurialErrorMessage($message);
throw new Exception($message);
}
}
}
/**
* @task hg
*/
private function executeMercurialUpdate() {
$repository = $this->getRepository();
$path = $repository->getLocalPath();
// This is a local command, but needs credentials.
$remote = $repository->getRemoteURIEnvelope();
$future = $repository->getRemoteCommandFuture('pull -u -- %P', $remote);
$future->setCWD($path);
try {
$future->resolvex();
} catch (CommandException $ex) {
$err = $ex->getError();
$stdout = $ex->getStdOut();
// NOTE: Between versions 2.1 and 2.1.1, Mercurial changed the behavior
// of "hg pull" to return 1 in case of a successful pull with no changes.
// This behavior has been reverted, but users who updated between Feb 1,
// 2012 and Mar 1, 2012 will have the erroring version. Do a dumb test
// against stdout to check for this possibility.
// See: https://github.com/phacility/phabricator/issues/101/
// NOTE: Mercurial has translated versions, which translate this error
// string. In a translated version, the string will be something else,
// like "aucun changement trouve". There didn't seem to be an easy way
// to handle this (there are hard ways but this is not a common problem
// and only creates log spam, not application failures). Assume English.
// TODO: Remove this once we're far enough in the future that deployment
// of 2.1 is exceedingly rare?
if ($err == 1 && preg_match('/no changes found/', $stdout)) {
return;
} else {
$message = $ex->getMessage();
$message = $this->censorMercurialErrorMessage($message);
throw new Exception($message);
}
}
}
/**
* Censor response bodies from Mercurial error messages.
*
* When Mercurial attempts to clone an HTTP repository but does not
* receive a response it expects, it emits the response body in the
* command output.
*
* This represents a potential SSRF issue, because an attacker with
* permission to create repositories can create one which points at the
* remote URI for some local service, then read the response from the
* error message. To prevent this, censor response bodies out of error
* messages.
*
* @param string Uncensored Mercurial command output.
* @return string Censored Mercurial command output.
*/
private function censorMercurialErrorMessage($message) {
return preg_replace(
'/^---%<---.*/sm',
pht('<Response body omitted from Mercurial error message.>')."\n",
$message);
}
/**
* @task hg
*/
private function installMercurialHook() {
$repository = $this->getRepository();
$path = $repository->getLocalPath().'/.hg/hgrc';
$identifier = $this->getHookContextIdentifier($repository);
$root = dirname(phutil_get_library_root('phabricator'));
$bin = $root.'/bin/commit-hook';
$data = array();
$data[] = '[hooks]';
// This hook handles normal pushes.
$data[] = csprintf(
'pretxnchangegroup.phabricator = TERM=dumb %s %s %s',
$bin,
$identifier,
'pretxnchangegroup');
// This one handles creating bookmarks.
$data[] = csprintf(
'prepushkey.phabricator = TERM=dumb %s %s %s',
$bin,
$identifier,
'prepushkey');
$data[] = null;
$data = implode("\n", $data);
$this->log('%s', pht('Installing commit hook config to "%s"...', $path));
Filesystem::writeFile($path, $data);
}
/* -( Pulling Subversion Working Copies )---------------------------------- */
/**
* @task svn
*/
private function executeSubversionCreate() {
$repository = $this->getRepository();
$path = rtrim($repository->getLocalPath(), '/');
execx('svnadmin create -- %s', $path);
}
/**
* @task svn
*/
private function installSubversionHook() {
$repository = $this->getRepository();
$root = $repository->getLocalPath();
$path = '/hooks/pre-commit';
$this->installHook($root.$path);
}
}
diff --git a/src/applications/repository/engine/PhabricatorRepositoryRefEngine.php b/src/applications/repository/engine/PhabricatorRepositoryRefEngine.php
index 3909cc83b0..da89db75e0 100644
--- a/src/applications/repository/engine/PhabricatorRepositoryRefEngine.php
+++ b/src/applications/repository/engine/PhabricatorRepositoryRefEngine.php
@@ -1,494 +1,500 @@
<?php
/**
* Update the ref cursors for a repository, which track the positions of
* branches, bookmarks, and tags.
*/
final class PhabricatorRepositoryRefEngine
extends PhabricatorRepositoryEngine {
private $newRefs = array();
private $deadRefs = array();
private $closeCommits = array();
private $hasNoCursors;
public function updateRefs() {
$this->newRefs = array();
$this->deadRefs = array();
$this->closeCommits = array();
$repository = $this->getRepository();
$branches_may_close = false;
$vcs = $repository->getVersionControlSystem();
switch ($vcs) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
// No meaningful refs of any type in Subversion.
$branches = array();
$bookmarks = array();
$tags = array();
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
$branches = $this->loadMercurialBranchPositions($repository);
$bookmarks = $this->loadMercurialBookmarkPositions($repository);
$tags = array();
$branches_may_close = true;
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
$branches = $this->loadGitBranchPositions($repository);
$bookmarks = array();
$tags = $this->loadGitTagPositions($repository);
break;
default:
throw new Exception(pht('Unknown VCS "%s"!', $vcs));
}
$maps = array(
PhabricatorRepositoryRefCursor::TYPE_BRANCH => $branches,
PhabricatorRepositoryRefCursor::TYPE_TAG => $tags,
PhabricatorRepositoryRefCursor::TYPE_BOOKMARK => $bookmarks,
);
$all_cursors = id(new PhabricatorRepositoryRefCursorQuery())
->setViewer(PhabricatorUser::getOmnipotentUser())
->withRepositoryPHIDs(array($repository->getPHID()))
->execute();
$cursor_groups = mgroup($all_cursors, 'getRefType');
$this->hasNoCursors = (!$all_cursors);
// Find all the heads of closing refs.
$all_closing_heads = array();
foreach ($all_cursors as $cursor) {
if ($this->shouldCloseRef($cursor->getRefType(), $cursor->getRefName())) {
$all_closing_heads[] = $cursor->getCommitIdentifier();
}
}
$all_closing_heads = array_unique($all_closing_heads);
$all_closing_heads = $this->removeMissingCommits($all_closing_heads);
foreach ($maps as $type => $refs) {
$cursor_group = idx($cursor_groups, $type, array());
$this->updateCursors($cursor_group, $refs, $type, $all_closing_heads);
}
if ($this->closeCommits) {
$this->setCloseFlagOnCommits($this->closeCommits);
}
if ($this->newRefs || $this->deadRefs) {
$repository->openTransaction();
foreach ($this->newRefs as $ref) {
$ref->save();
}
foreach ($this->deadRefs as $ref) {
$ref->delete();
}
$repository->saveTransaction();
$this->newRefs = array();
$this->deadRefs = array();
}
if ($branches && $branches_may_close) {
$this->updateBranchStates($repository, $branches);
}
}
private function updateBranchStates(
PhabricatorRepository $repository,
array $branches) {
assert_instances_of($branches, 'DiffusionRepositoryRef');
$all_cursors = id(new PhabricatorRepositoryRefCursorQuery())
->setViewer(PhabricatorUser::getOmnipotentUser())
->withRepositoryPHIDs(array($repository->getPHID()))
->execute();
$state_map = array();
$type_branch = PhabricatorRepositoryRefCursor::TYPE_BRANCH;
foreach ($all_cursors as $cursor) {
if ($cursor->getRefType() !== $type_branch) {
continue;
}
$raw_name = $cursor->getRefNameRaw();
$hash = $cursor->getCommitIdentifier();
$state_map[$raw_name][$hash] = $cursor;
}
foreach ($branches as $branch) {
$cursor = idx($state_map, $branch->getShortName(), array());
$cursor = idx($cursor, $branch->getCommitIdentifier());
if (!$cursor) {
continue;
}
$fields = $branch->getRawFields();
$cursor_state = (bool)$cursor->getIsClosed();
$branch_state = (bool)idx($fields, 'closed');
if ($cursor_state != $branch_state) {
$cursor->setIsClosed((int)$branch_state)->save();
}
}
}
private function markRefNew(PhabricatorRepositoryRefCursor $cursor) {
$this->newRefs[] = $cursor;
return $this;
}
private function markRefDead(PhabricatorRepositoryRefCursor $cursor) {
$this->deadRefs[] = $cursor;
return $this;
}
private function markCloseCommits(array $identifiers) {
foreach ($identifiers as $identifier) {
$this->closeCommits[$identifier] = $identifier;
}
return $this;
}
/**
* Remove commits which no longer exist in the repository from a list.
*
* After a force push and garbage collection, we may have branch cursors which
* point at commits which no longer exist. This can make commands issued later
* fail. See T5839 for discussion.
*
* @param list<string> List of commit identifiers.
* @return list<string> List with nonexistent identifiers removed.
*/
private function removeMissingCommits(array $identifiers) {
if (!$identifiers) {
return array();
}
$resolved = id(new DiffusionLowLevelResolveRefsQuery())
->setRepository($this->getRepository())
->withRefs($identifiers)
->execute();
foreach ($identifiers as $key => $identifier) {
if (empty($resolved[$identifier])) {
unset($identifiers[$key]);
}
}
return $identifiers;
}
private function updateCursors(
array $cursors,
array $new_refs,
$ref_type,
array $all_closing_heads) {
$repository = $this->getRepository();
// NOTE: Mercurial branches may have multiple branch heads; this logic
// is complex primarily to account for that.
// Group all the cursors by their ref name, like "master". Since Mercurial
// branches may have multiple heads, there could be several cursors with
// the same name.
$cursor_groups = mgroup($cursors, 'getRefNameRaw');
// Group all the new ref values by their name. As above, these groups may
// have multiple members in Mercurial.
$ref_groups = mgroup($new_refs, 'getShortName');
foreach ($ref_groups as $name => $refs) {
$new_commits = mpull($refs, 'getCommitIdentifier', 'getCommitIdentifier');
$ref_cursors = idx($cursor_groups, $name, array());
$old_commits = mpull($ref_cursors, null, 'getCommitIdentifier');
// We're going to delete all the cursors pointing at commits which are
// no longer associated with the refs. This primarily makes the Mercurial
// multiple head case easier, and means that when we update a ref we
// delete the old one and write a new one.
foreach ($ref_cursors as $cursor) {
if (isset($new_commits[$cursor->getCommitIdentifier()])) {
// This ref previously pointed at this commit, and still does.
$this->log(
pht(
'Ref %s "%s" still points at %s.',
$ref_type,
$name,
$cursor->getCommitIdentifier()));
} else {
// This ref previously pointed at this commit, but no longer does.
$this->log(
pht(
'Ref %s "%s" no longer points at %s.',
$ref_type,
$name,
$cursor->getCommitIdentifier()));
// Nuke the obsolete cursor.
$this->markRefDead($cursor);
}
}
// Now, we're going to insert new cursors for all the commits which are
// associated with this ref that don't currently have cursors.
$added_commits = array_diff_key($new_commits, $old_commits);
foreach ($added_commits as $identifier) {
$this->log(
pht(
'Ref %s "%s" now points at %s.',
$ref_type,
$name,
$identifier));
$this->markRefNew(
id(new PhabricatorRepositoryRefCursor())
->setRepositoryPHID($repository->getPHID())
->setRefType($ref_type)
->setRefName($name)
->setCommitIdentifier($identifier));
}
if ($this->shouldCloseRef($ref_type, $name)) {
foreach ($added_commits as $identifier) {
$new_identifiers = $this->loadNewCommitIdentifiers(
$identifier,
$all_closing_heads);
$this->markCloseCommits($new_identifiers);
}
}
}
// Find any cursors for refs which no longer exist. This happens when a
// branch, tag or bookmark is deleted.
foreach ($cursor_groups as $name => $cursor_group) {
if (idx($ref_groups, $name) === null) {
foreach ($cursor_group as $cursor) {
$this->log(
pht(
'Ref %s "%s" no longer exists.',
$cursor->getRefType(),
$cursor->getRefName()));
$this->markRefDead($cursor);
}
}
}
}
private function shouldCloseRef($ref_type, $ref_name) {
if ($ref_type !== PhabricatorRepositoryRefCursor::TYPE_BRANCH) {
return false;
}
if ($this->hasNoCursors) {
// If we don't have any cursors, don't close things. Particularly, this
// corresponds to the case where you've just updated to this code on an
// existing repository: we don't want to requeue message steps for every
// commit on a closeable ref.
return false;
}
return $this->getRepository()->shouldAutocloseBranch($ref_name);
}
/**
* Find all ancestors of a new closing branch head which are not ancestors
* of any old closing branch head.
*/
private function loadNewCommitIdentifiers(
$new_head,
array $all_closing_heads) {
$repository = $this->getRepository();
$vcs = $repository->getVersionControlSystem();
switch ($vcs) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
if ($all_closing_heads) {
$parts = array();
foreach ($all_closing_heads as $head) {
$parts[] = hgsprintf('%s', $head);
}
// See T5896. Mercurial can not parse an "X or Y or ..." rev list
// with more than about 300 items, because it exceeds the maximum
// allowed recursion depth. Split all the heads into chunks of
// 256, and build a query like this:
//
// ((1 or 2 or ... or 255) or (256 or 257 or ... 511))
//
// If we have more than 65535 heads, we'll do that again:
//
// (((1 or ...) or ...) or ((65536 or ...) or ...))
$chunk_size = 256;
while (count($parts) > $chunk_size) {
$chunks = array_chunk($parts, $chunk_size);
foreach ($chunks as $key => $chunk) {
$chunks[$key] = '('.implode(' or ', $chunk).')';
}
$parts = array_values($chunks);
}
$parts = '('.implode(' or ', $parts).')';
list($stdout) = $this->getRepository()->execxLocalCommand(
'log --template %s --rev %s',
'{node}\n',
hgsprintf('%s', $new_head).' - '.$parts);
} else {
list($stdout) = $this->getRepository()->execxLocalCommand(
'log --template %s --rev %s',
'{node}\n',
hgsprintf('%s', $new_head));
}
$stdout = trim($stdout);
if (!strlen($stdout)) {
return array();
}
return phutil_split_lines($stdout, $retain_newlines = false);
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
if ($all_closing_heads) {
list($stdout) = $this->getRepository()->execxLocalCommand(
'log --format=%s %s --not %Ls',
'%H',
$new_head,
$all_closing_heads);
} else {
list($stdout) = $this->getRepository()->execxLocalCommand(
'log --format=%s %s',
'%H',
$new_head);
}
$stdout = trim($stdout);
if (!strlen($stdout)) {
return array();
}
return phutil_split_lines($stdout, $retain_newlines = false);
default:
throw new Exception(pht('Unsupported VCS "%s"!', $vcs));
}
}
/**
* Mark a list of commits as closeable, and queue workers for those commits
* which don't already have the flag.
*/
private function setCloseFlagOnCommits(array $identifiers) {
$repository = $this->getRepository();
$commit_table = new PhabricatorRepositoryCommit();
$conn_w = $commit_table->establishConnection('w');
$vcs = $repository->getVersionControlSystem();
switch ($vcs) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
$class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
$class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
$class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker';
break;
default:
throw new Exception(pht("Unknown repository type '%s'!", $vcs));
}
$all_commits = queryfx_all(
$conn_w,
'SELECT id, commitIdentifier, importStatus FROM %T
WHERE repositoryID = %d AND commitIdentifier IN (%Ls)',
$commit_table->getTableName(),
$repository->getID(),
$identifiers);
$closeable_flag = PhabricatorRepositoryCommit::IMPORTED_CLOSEABLE;
$all_commits = ipull($all_commits, null, 'commitIdentifier');
foreach ($identifiers as $identifier) {
$row = idx($all_commits, $identifier);
if (!$row) {
throw new Exception(
pht(
'Commit "%s" has not been discovered yet! Run discovery before '.
'updating refs.',
$identifier));
}
if (!($row['importStatus'] & $closeable_flag)) {
queryfx(
$conn_w,
'UPDATE %T SET importStatus = (importStatus | %d) WHERE id = %d',
$commit_table->getTableName(),
$closeable_flag,
$row['id']);
$data = array(
'commitID' => $row['id'],
'only' => true,
);
PhabricatorWorker::scheduleTask($class, $data);
}
}
return $this;
}
/* -( Updating Git Refs )-------------------------------------------------- */
/**
* @task git
*/
private function loadGitBranchPositions(PhabricatorRepository $repository) {
return id(new DiffusionLowLevelGitRefQuery())
->setRepository($repository)
- ->withIsOriginBranch(true)
+ ->withRefTypes(
+ array(
+ PhabricatorRepositoryRefCursor::TYPE_BRANCH,
+ ))
->execute();
}
/**
* @task git
*/
private function loadGitTagPositions(PhabricatorRepository $repository) {
return id(new DiffusionLowLevelGitRefQuery())
->setRepository($repository)
- ->withIsTag(true)
+ ->withRefTypes(
+ array(
+ PhabricatorRepositoryRefCursor::TYPE_TAG,
+ ))
->execute();
}
/* -( Updating Mercurial Refs )-------------------------------------------- */
/**
* @task hg
*/
private function loadMercurialBranchPositions(
PhabricatorRepository $repository) {
return id(new DiffusionLowLevelMercurialBranchesQuery())
->setRepository($repository)
->execute();
}
/**
* @task hg
*/
private function loadMercurialBookmarkPositions(
PhabricatorRepository $repository) {
// TODO: Implement support for Mercurial bookmarks.
return array();
}
}

File Metadata

Mime Type
text/x-diff
Expires
Tue, Apr 29, 2:02 AM (19 h, 5 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
108156
Default Alt Text
(127 KB)

Event Timeline

styx copyright 2020 sirocyl / ellie (kymkdd) / ncommander / styx contributors
Use of other names and brands is not intended to reflect a claim of, or right to, use the name or brand.
styx is an operating system using the Linux kernel and components, and "styx Linux" as a phrase does not reflect the inclusion of the Linux brand or trademark in the styx project.
Linux® is the registered trademark of Linus Torvalds in the U.S. and other countries.
Terms and Conditions (TODO: fix this!)