Page MenuHomestyx hydra
Files F1457665

No OneTemporary
Actions

View Options

diff --git a/src/applications/config/check/PhabricatorStorageSetupCheck.php b/src/applications/config/check/PhabricatorStorageSetupCheck.php
index 86aa1f2ebd..75639f2f43 100644
--- a/src/applications/config/check/PhabricatorStorageSetupCheck.php
+++ b/src/applications/config/check/PhabricatorStorageSetupCheck.php
@@ -1,196 +1,196 @@
<?php
final class PhabricatorStorageSetupCheck extends PhabricatorSetupCheck {
public function getDefaultGroup() {
return self::GROUP_OTHER;
}
/**
* @phutil-external-symbol class PhabricatorStartup
*/
protected function executeChecks() {
$engines = PhabricatorFileStorageEngine::loadWritableChunkEngines();
$chunk_engine_active = (bool)$engines;
$this->checkS3();
if (!$chunk_engine_active) {
$doc_href = PhabricatorEnv::getDoclink('Configuring File Storage');
$message = pht(
'Large file storage has not been configured, which will limit '.
'the maximum size of file uploads. See %s for '.
'instructions on configuring uploads and storage.',
phutil_tag(
'a',
array(
'href' => $doc_href,
'target' => '_blank',
),
pht('Configuring File Storage')));
$this
->newIssue('large-files')
->setShortName(pht('Large Files'))
->setName(pht('Large File Storage Not Configured'))
->setMessage($message);
}
$post_max_size = ini_get('post_max_size');
if ($post_max_size && ((int)$post_max_size > 0)) {
$post_max_bytes = phutil_parse_bytes($post_max_size);
$post_max_need = (32 * 1024 * 1024);
if ($post_max_need > $post_max_bytes) {
$summary = pht(
'Set %s in your PHP configuration to at least 32MB '.
'to support large file uploads.',
phutil_tag('tt', array(), 'post_max_size'));
$message = pht(
'Adjust %s in your PHP configuration to at least 32MB. When '.
'set to smaller value, large file uploads may not work properly.',
phutil_tag('tt', array(), 'post_max_size'));
$this
->newIssue('php.post_max_size')
->setName(pht('PHP post_max_size Not Configured'))
->setSummary($summary)
->setMessage($message)
->setGroup(self::GROUP_PHP)
->addPHPConfig('post_max_size');
}
}
// This is somewhat arbitrary, but make sure we have enough headroom to
// upload a default file at the chunk threshold (8MB), which may be
// base64 encoded, then JSON encoded in the request, and may need to be
// held in memory in the raw and as a query string.
$need_bytes = (64 * 1024 * 1024);
$memory_limit = PhabricatorStartup::getOldMemoryLimit();
if ($memory_limit && ((int)$memory_limit > 0)) {
$memory_limit_bytes = phutil_parse_bytes($memory_limit);
$memory_usage_bytes = memory_get_usage();
$available_bytes = ($memory_limit_bytes - $memory_usage_bytes);
if ($need_bytes > $available_bytes) {
$summary = pht(
'Your PHP memory limit is configured in a way that may prevent '.
'you from uploading large files or handling large requests.');
$message = pht(
'When you upload a file via drag-and-drop or the API, chunks must '.
'be buffered into memory before being written to permanent '.
'storage. This server needs memory available to store these '.
'chunks while they are uploaded, but PHP is currently configured '.
'to severely limit the available memory.'.
"\n\n".
'PHP processes currently have very little free memory available '.
'(%s). To work well, processes should have at least %s.'.
"\n\n".
'(Note that the application itself must also fit in available '.
'memory, so not all of the memory under the memory limit is '.
'available for running workloads.)'.
"\n\n".
"The easiest way to resolve this issue is to set %s to %s in your ".
"PHP configuration, to disable the memory limit. There is ".
"usually little or no value to using this option to limit ".
"process memory.".
"\n\n".
"You can also increase the limit or ignore this issue and accept ".
"that you may encounter problems uploading large files and ".
"processing large requests.",
phutil_format_bytes($available_bytes),
phutil_format_bytes($need_bytes),
phutil_tag('tt', array(), 'memory_limit'),
phutil_tag('tt', array(), '-1'));
$this
->newIssue('php.memory_limit.upload')
->setName(pht('Memory Limit Restricts File Uploads'))
->setSummary($summary)
->setMessage($message)
->setGroup(self::GROUP_PHP)
->addPHPConfig('memory_limit')
->addPHPConfigOriginalValue('memory_limit', $memory_limit);
}
}
$local_path = PhabricatorEnv::getEnvConfig('storage.local-disk.path');
if (!$local_path) {
return;
}
if (!Filesystem::pathExists($local_path) ||
!is_readable($local_path) ||
!is_writable($local_path)) {
$message = pht(
'Configured location for storing uploaded files on disk ("%s") does '.
'not exist, or is not readable or writable. Verify the directory '.
'exists and is readable and writable by the webserver.',
$local_path);
$this
->newIssue('config.storage.local-disk.path')
->setShortName(pht('Local Disk Storage'))
->setName(pht('Local Disk Storage Not Readable/Writable'))
->setMessage($message)
->addPhabricatorConfig('storage.local-disk.path');
}
}
private function checkS3() {
$access_key = PhabricatorEnv::getEnvConfig('amazon-s3.access-key');
$secret_key = PhabricatorEnv::getEnvConfig('amazon-s3.secret-key');
$region = PhabricatorEnv::getEnvConfig('amazon-s3.region');
$endpoint = PhabricatorEnv::getEnvConfig('amazon-s3.endpoint');
$how_many = 0;
- if (strlen($access_key)) {
+ if (phutil_nonempty_string($access_key)) {
$how_many++;
}
- if (strlen($secret_key)) {
+ if (phutil_nonempty_string($secret_key)) {
$how_many++;
}
- if (strlen($region)) {
+ if (phutil_nonempty_string($region)) {
$how_many++;
}
- if (strlen($endpoint)) {
+ if (phutil_nonempty_string($endpoint)) {
$how_many++;
}
// Nothing configured, no issues here.
if ($how_many === 0) {
return;
}
// Everything configured, no issues here.
if ($how_many === 4) {
return;
}
$message = pht(
'File storage in Amazon S3 has been partially configured, but you are '.
'missing some required settings. S3 will not be available to store '.
'files until you complete the configuration. Either configure S3 fully '.
'or remove the partial configuration.');
$this->newIssue('storage.s3.partial-config')
->setShortName(pht('S3 Partially Configured'))
->setName(pht('Amazon S3 is Only Partially Configured'))
->setMessage($message)
->addPhabricatorConfig('amazon-s3.access-key')
->addPhabricatorConfig('amazon-s3.secret-key')
->addPhabricatorConfig('amazon-s3.region')
->addPhabricatorConfig('amazon-s3.endpoint');
}
}
diff --git a/src/applications/files/engine/PhabricatorS3FileStorageEngine.php b/src/applications/files/engine/PhabricatorS3FileStorageEngine.php
index 443a5a9ddc..95cdfc737b 100644
--- a/src/applications/files/engine/PhabricatorS3FileStorageEngine.php
+++ b/src/applications/files/engine/PhabricatorS3FileStorageEngine.php
@@ -1,171 +1,171 @@
<?php
/**
* Amazon S3 file storage engine. This engine scales well but is relatively
* high-latency since data has to be pulled off S3.
*
* @task internal Internals
*/
final class PhabricatorS3FileStorageEngine
extends PhabricatorFileStorageEngine {
/* -( Engine Metadata )---------------------------------------------------- */
/**
* This engine identifies as `amazon-s3`.
*/
public function getEngineIdentifier() {
return 'amazon-s3';
}
public function getEnginePriority() {
return 100;
}
public function canWriteFiles() {
$bucket = PhabricatorEnv::getEnvConfig('storage.s3.bucket');
$access_key = PhabricatorEnv::getEnvConfig('amazon-s3.access-key');
$secret_key = PhabricatorEnv::getEnvConfig('amazon-s3.secret-key');
$endpoint = PhabricatorEnv::getEnvConfig('amazon-s3.endpoint');
$region = PhabricatorEnv::getEnvConfig('amazon-s3.region');
- return (strlen($bucket) &&
- strlen($access_key) &&
- strlen($secret_key) &&
- strlen($endpoint) &&
- strlen($region));
+ return phutil_nonempty_string($bucket) &&
+ phutil_nonempty_string($access_key) &&
+ phutil_nonempty_string($secret_key) &&
+ phutil_nonempty_string($endpoint) &&
+ phutil_nonempty_string($region);
}
/* -( Managing File Data )------------------------------------------------- */
/**
* Writes file data into Amazon S3.
*/
public function writeFile($data, array $params) {
$s3 = $this->newS3API();
// Generate a random name for this file. We add some directories to it
// (e.g. 'abcdef123456' becomes 'ab/cd/ef123456') to make large numbers of
// files more browsable with web/debugging tools like the S3 administration
// tool.
$seed = Filesystem::readRandomCharacters(20);
$parts = array();
$parts[] = 'phabricator';
$instance_name = PhabricatorEnv::getEnvConfig('cluster.instance');
- if (strlen($instance_name)) {
+ if (phutil_nonempty_string($instance_name)) {
$parts[] = $instance_name;
}
$parts[] = substr($seed, 0, 2);
$parts[] = substr($seed, 2, 2);
$parts[] = substr($seed, 4);
$name = implode('/', $parts);
AphrontWriteGuard::willWrite();
$profiler = PhutilServiceProfiler::getInstance();
$call_id = $profiler->beginServiceCall(
array(
'type' => 's3',
'method' => 'putObject',
));
$s3
->setParametersForPutObject($name, $data)
->resolve();
$profiler->endServiceCall($call_id, array());
return $name;
}
/**
* Load a stored blob from Amazon S3.
*/
public function readFile($handle) {
$s3 = $this->newS3API();
$profiler = PhutilServiceProfiler::getInstance();
$call_id = $profiler->beginServiceCall(
array(
'type' => 's3',
'method' => 'getObject',
));
$result = $s3
->setParametersForGetObject($handle)
->resolve();
$profiler->endServiceCall($call_id, array());
return $result;
}
/**
* Delete a blob from Amazon S3.
*/
public function deleteFile($handle) {
$s3 = $this->newS3API();
AphrontWriteGuard::willWrite();
$profiler = PhutilServiceProfiler::getInstance();
$call_id = $profiler->beginServiceCall(
array(
'type' => 's3',
'method' => 'deleteObject',
));
$s3
->setParametersForDeleteObject($handle)
->resolve();
$profiler->endServiceCall($call_id, array());
}
/* -( Internals )---------------------------------------------------------- */
/**
* Retrieve the S3 bucket name.
*
* @task internal
*/
private function getBucketName() {
$bucket = PhabricatorEnv::getEnvConfig('storage.s3.bucket');
if (!$bucket) {
throw new PhabricatorFileStorageConfigurationException(
pht(
"No '%s' specified!",
'storage.s3.bucket'));
}
return $bucket;
}
/**
* Create a new S3 API object.
*
* @task internal
*/
private function newS3API() {
$access_key = PhabricatorEnv::getEnvConfig('amazon-s3.access-key');
$secret_key = PhabricatorEnv::getEnvConfig('amazon-s3.secret-key');
$region = PhabricatorEnv::getEnvConfig('amazon-s3.region');
$endpoint = PhabricatorEnv::getEnvConfig('amazon-s3.endpoint');
return id(new PhutilAWSS3Future())
->setAccessKey($access_key)
->setSecretKey(new PhutilOpaqueEnvelope($secret_key))
->setRegion($region)
->setEndpoint($endpoint)
->setBucket($this->getBucketName());
}
}
diff --git a/src/applications/files/storage/PhabricatorFile.php b/src/applications/files/storage/PhabricatorFile.php
index cc1b701dcd..355aa2d215 100644
--- a/src/applications/files/storage/PhabricatorFile.php
+++ b/src/applications/files/storage/PhabricatorFile.php
@@ -1,1764 +1,1764 @@
<?php
/**
* Parameters
* ==========
*
* When creating a new file using a method like @{method:newFromFileData}, these
* parameters are supported:
*
* | name | Human readable filename.
* | authorPHID | User PHID of uploader.
* | ttl.absolute | Temporary file lifetime as an epoch timestamp.
* | ttl.relative | Temporary file lifetime, relative to now, in seconds.
* | viewPolicy | File visibility policy.
* | isExplicitUpload | Used to show users files they explicitly uploaded.
* | canCDN | Allows the file to be cached and delivered over a CDN.
* | profile | Marks the file as a profile image.
* | format | Internal encoding format.
* | mime-type | Optional, explicit file MIME type.
* | builtin | Optional filename, identifies this as a builtin.
*
*/
final class PhabricatorFile extends PhabricatorFileDAO
implements
PhabricatorApplicationTransactionInterface,
PhabricatorTokenReceiverInterface,
PhabricatorSubscribableInterface,
PhabricatorFlaggableInterface,
PhabricatorPolicyInterface,
PhabricatorDestructibleInterface,
PhabricatorConduitResultInterface,
PhabricatorIndexableInterface,
PhabricatorNgramsInterface {
const METADATA_IMAGE_WIDTH = 'width';
const METADATA_IMAGE_HEIGHT = 'height';
const METADATA_CAN_CDN = 'canCDN';
const METADATA_BUILTIN = 'builtin';
const METADATA_PARTIAL = 'partial';
const METADATA_PROFILE = 'profile';
const METADATA_STORAGE = 'storage';
const METADATA_INTEGRITY = 'integrity';
const METADATA_CHUNK = 'chunk';
const METADATA_ALT_TEXT = 'alt';
const STATUS_ACTIVE = 'active';
const STATUS_DELETED = 'deleted';
protected $name;
protected $mimeType;
protected $byteSize;
protected $authorPHID;
protected $secretKey;
protected $contentHash;
protected $metadata = array();
protected $mailKey;
protected $builtinKey;
protected $storageEngine;
protected $storageFormat;
protected $storageHandle;
protected $ttl;
protected $isExplicitUpload = 1;
protected $viewPolicy = PhabricatorPolicies::POLICY_USER;
protected $isPartial = 0;
protected $isDeleted = 0;
private $objects = self::ATTACHABLE;
private $objectPHIDs = self::ATTACHABLE;
private $originalFile = self::ATTACHABLE;
private $transforms = self::ATTACHABLE;
public static function initializeNewFile() {
$app = id(new PhabricatorApplicationQuery())
->setViewer(PhabricatorUser::getOmnipotentUser())
->withClasses(array('PhabricatorFilesApplication'))
->executeOne();
$view_policy = $app->getPolicy(
FilesDefaultViewCapability::CAPABILITY);
return id(new PhabricatorFile())
->setViewPolicy($view_policy)
->setIsPartial(0)
->attachOriginalFile(null)
->attachObjects(array())
->attachObjectPHIDs(array());
}
protected function getConfiguration() {
return array(
self::CONFIG_AUX_PHID => true,
self::CONFIG_SERIALIZATION => array(
'metadata' => self::SERIALIZATION_JSON,
),
self::CONFIG_COLUMN_SCHEMA => array(
'name' => 'sort255?',
'mimeType' => 'text255?',
'byteSize' => 'uint64',
'storageEngine' => 'text32',
'storageFormat' => 'text32',
'storageHandle' => 'text255',
'authorPHID' => 'phid?',
'secretKey' => 'bytes20?',
'contentHash' => 'bytes64?',
'ttl' => 'epoch?',
'isExplicitUpload' => 'bool?',
'mailKey' => 'bytes20',
'isPartial' => 'bool',
'builtinKey' => 'text64?',
'isDeleted' => 'bool',
),
self::CONFIG_KEY_SCHEMA => array(
'key_phid' => null,
'phid' => array(
'columns' => array('phid'),
'unique' => true,
),
'authorPHID' => array(
'columns' => array('authorPHID'),
),
'contentHash' => array(
'columns' => array('contentHash'),
),
'key_ttl' => array(
'columns' => array('ttl'),
),
'key_dateCreated' => array(
'columns' => array('dateCreated'),
),
'key_partial' => array(
'columns' => array('authorPHID', 'isPartial'),
),
'key_builtin' => array(
'columns' => array('builtinKey'),
'unique' => true,
),
'key_engine' => array(
'columns' => array('storageEngine', 'storageHandle(64)'),
),
),
) + parent::getConfiguration();
}
public function generatePHID() {
return PhabricatorPHID::generateNewPHID(
PhabricatorFileFilePHIDType::TYPECONST);
}
public function save() {
if (!$this->getSecretKey()) {
$this->setSecretKey($this->generateSecretKey());
}
if (!$this->getMailKey()) {
$this->setMailKey(Filesystem::readRandomCharacters(20));
}
return parent::save();
}
public function saveAndIndex() {
$this->save();
if ($this->isIndexableFile()) {
PhabricatorSearchWorker::queueDocumentForIndexing($this->getPHID());
}
return $this;
}
private function isIndexableFile() {
if ($this->getIsChunk()) {
return false;
}
return true;
}
public function getMonogram() {
return 'F'.$this->getID();
}
public function scrambleSecret() {
return $this->setSecretKey($this->generateSecretKey());
}
public static function readUploadedFileData($spec) {
if (!$spec) {
throw new Exception(pht('No file was uploaded!'));
}
$err = idx($spec, 'error');
if ($err) {
throw new PhabricatorFileUploadException($err);
}
$tmp_name = idx($spec, 'tmp_name');
// NOTE: If we parsed the request body ourselves, the files we wrote will
// not be registered in the `is_uploaded_file()` list. It's fine to skip
// this check: it just protects against sloppy code from the long ago era
// of "register_globals".
if (ini_get('enable_post_data_reading')) {
$is_valid = @is_uploaded_file($tmp_name);
if (!$is_valid) {
throw new Exception(pht('File is not an uploaded file.'));
}
}
$file_data = Filesystem::readFile($tmp_name);
$file_size = idx($spec, 'size');
if (strlen($file_data) != $file_size) {
throw new Exception(pht('File size disagrees with uploaded size.'));
}
return $file_data;
}
public static function newFromPHPUpload($spec, array $params = array()) {
$file_data = self::readUploadedFileData($spec);
$file_name = nonempty(
idx($params, 'name'),
idx($spec, 'name'));
$params = array(
'name' => $file_name,
) + $params;
return self::newFromFileData($file_data, $params);
}
public static function newFromXHRUpload($data, array $params = array()) {
return self::newFromFileData($data, $params);
}
public static function newFileFromContentHash($hash, array $params) {
if ($hash === null) {
return null;
}
// Check to see if a file with same hash already exists.
$file = id(new PhabricatorFile())->loadOneWhere(
'contentHash = %s LIMIT 1',
$hash);
if (!$file) {
return null;
}
$copy_of_storage_engine = $file->getStorageEngine();
$copy_of_storage_handle = $file->getStorageHandle();
$copy_of_storage_format = $file->getStorageFormat();
$copy_of_storage_properties = $file->getStorageProperties();
$copy_of_byte_size = $file->getByteSize();
$copy_of_mime_type = $file->getMimeType();
$new_file = self::initializeNewFile();
$new_file->setByteSize($copy_of_byte_size);
$new_file->setContentHash($hash);
$new_file->setStorageEngine($copy_of_storage_engine);
$new_file->setStorageHandle($copy_of_storage_handle);
$new_file->setStorageFormat($copy_of_storage_format);
$new_file->setStorageProperties($copy_of_storage_properties);
$new_file->setMimeType($copy_of_mime_type);
$new_file->copyDimensions($file);
$new_file->readPropertiesFromParameters($params);
$new_file->saveAndIndex();
return $new_file;
}
public static function newChunkedFile(
PhabricatorFileStorageEngine $engine,
$length,
array $params) {
$file = self::initializeNewFile();
$file->setByteSize($length);
// NOTE: Once we receive the first chunk, we'll detect its MIME type and
// update the parent file if a MIME type hasn't been provided. This matters
// for large media files like video.
$mime_type = idx($params, 'mime-type');
if (!strlen($mime_type)) {
$file->setMimeType('application/octet-stream');
}
$chunked_hash = idx($params, 'chunkedHash');
// Get rid of this parameter now; we aren't passing it any further down
// the stack.
unset($params['chunkedHash']);
if ($chunked_hash) {
$file->setContentHash($chunked_hash);
} else {
// See PhabricatorChunkedFileStorageEngine::getChunkedHash() for some
// discussion of this.
$seed = Filesystem::readRandomBytes(64);
$hash = PhabricatorChunkedFileStorageEngine::getChunkedHashForInput(
$seed);
$file->setContentHash($hash);
}
$file->setStorageEngine($engine->getEngineIdentifier());
$file->setStorageHandle(PhabricatorFileChunk::newChunkHandle());
// Chunked files are always stored raw because they do not actually store
// data. The chunks do, and can be individually formatted.
$file->setStorageFormat(PhabricatorFileRawStorageFormat::FORMATKEY);
$file->setIsPartial(1);
$file->readPropertiesFromParameters($params);
return $file;
}
private static function buildFromFileData($data, array $params = array()) {
if (isset($params['storageEngines'])) {
$engines = $params['storageEngines'];
} else {
$size = strlen($data);
$engines = PhabricatorFileStorageEngine::loadStorageEngines($size);
if (!$engines) {
throw new Exception(
pht(
'No configured storage engine can store this file. See '.
'"Configuring File Storage" in the documentation for '.
'information on configuring storage engines.'));
}
}
assert_instances_of($engines, 'PhabricatorFileStorageEngine');
if (!$engines) {
throw new Exception(pht('No valid storage engines are available!'));
}
$file = self::initializeNewFile();
$aes_type = PhabricatorFileAES256StorageFormat::FORMATKEY;
$has_aes = PhabricatorKeyring::getDefaultKeyName($aes_type);
if ($has_aes !== null) {
$default_key = PhabricatorFileAES256StorageFormat::FORMATKEY;
} else {
$default_key = PhabricatorFileRawStorageFormat::FORMATKEY;
}
$key = idx($params, 'format', $default_key);
// Callers can pass in an object explicitly instead of a key. This is
// primarily useful for unit tests.
if ($key instanceof PhabricatorFileStorageFormat) {
$format = clone $key;
} else {
$format = clone PhabricatorFileStorageFormat::requireFormat($key);
}
$format->setFile($file);
$properties = $format->newStorageProperties();
$file->setStorageFormat($format->getStorageFormatKey());
$file->setStorageProperties($properties);
$data_handle = null;
$engine_identifier = null;
$integrity_hash = null;
$exceptions = array();
foreach ($engines as $engine) {
$engine_class = get_class($engine);
try {
$result = $file->writeToEngine(
$engine,
$data,
$params);
list($engine_identifier, $data_handle, $integrity_hash) = $result;
// We stored the file somewhere so stop trying to write it to other
// places.
break;
} catch (PhabricatorFileStorageConfigurationException $ex) {
// If an engine is outright misconfigured (or misimplemented), raise
// that immediately since it probably needs attention.
throw $ex;
} catch (Exception $ex) {
phlog($ex);
// If an engine doesn't work, keep trying all the other valid engines
// in case something else works.
$exceptions[$engine_class] = $ex;
}
}
if (!$data_handle) {
throw new PhutilAggregateException(
pht('All storage engines failed to write file:'),
$exceptions);
}
$file->setByteSize(strlen($data));
$hash = self::hashFileContent($data);
$file->setContentHash($hash);
$file->setStorageEngine($engine_identifier);
$file->setStorageHandle($data_handle);
$file->setIntegrityHash($integrity_hash);
$file->readPropertiesFromParameters($params);
if (!$file->getMimeType()) {
$tmp = new TempFile();
Filesystem::writeFile($tmp, $data);
$file->setMimeType(Filesystem::getMimeType($tmp));
unset($tmp);
}
try {
$file->updateDimensions(false);
} catch (Exception $ex) {
// Do nothing.
}
$file->saveAndIndex();
return $file;
}
public static function newFromFileData($data, array $params = array()) {
$hash = self::hashFileContent($data);
if ($hash !== null) {
$file = self::newFileFromContentHash($hash, $params);
if ($file) {
return $file;
}
}
return self::buildFromFileData($data, $params);
}
public function migrateToEngine(
PhabricatorFileStorageEngine $engine,
$make_copy) {
if (!$this->getID() || !$this->getStorageHandle()) {
throw new Exception(
pht("You can not migrate a file which hasn't yet been saved."));
}
$data = $this->loadFileData();
$params = array(
'name' => $this->getName(),
);
list($new_identifier, $new_handle, $integrity_hash) = $this->writeToEngine(
$engine,
$data,
$params);
$old_engine = $this->instantiateStorageEngine();
$old_identifier = $this->getStorageEngine();
$old_handle = $this->getStorageHandle();
$this->setStorageEngine($new_identifier);
$this->setStorageHandle($new_handle);
$this->setIntegrityHash($integrity_hash);
$this->save();
if (!$make_copy) {
$this->deleteFileDataIfUnused(
$old_engine,
$old_identifier,
$old_handle);
}
return $this;
}
public function migrateToStorageFormat(PhabricatorFileStorageFormat $format) {
if (!$this->getID() || !$this->getStorageHandle()) {
throw new Exception(
pht("You can not migrate a file which hasn't yet been saved."));
}
$data = $this->loadFileData();
$params = array(
'name' => $this->getName(),
);
$engine = $this->instantiateStorageEngine();
$old_handle = $this->getStorageHandle();
$properties = $format->newStorageProperties();
$this->setStorageFormat($format->getStorageFormatKey());
$this->setStorageProperties($properties);
list($identifier, $new_handle, $integrity_hash) = $this->writeToEngine(
$engine,
$data,
$params);
$this->setStorageHandle($new_handle);
$this->setIntegrityHash($integrity_hash);
$this->save();
$this->deleteFileDataIfUnused(
$engine,
$identifier,
$old_handle);
return $this;
}
public function cycleMasterStorageKey(PhabricatorFileStorageFormat $format) {
if (!$this->getID() || !$this->getStorageHandle()) {
throw new Exception(
pht("You can not cycle keys for a file which hasn't yet been saved."));
}
$properties = $format->cycleStorageProperties();
$this->setStorageProperties($properties);
$this->save();
return $this;
}
private function writeToEngine(
PhabricatorFileStorageEngine $engine,
$data,
array $params) {
$engine_class = get_class($engine);
$format = $this->newStorageFormat();
$data_iterator = array($data);
$formatted_iterator = $format->newWriteIterator($data_iterator);
$formatted_data = $this->loadDataFromIterator($formatted_iterator);
$integrity_hash = $engine->newIntegrityHash($formatted_data, $format);
$data_handle = $engine->writeFile($formatted_data, $params);
if (!$data_handle || strlen($data_handle) > 255) {
// This indicates an improperly implemented storage engine.
throw new PhabricatorFileStorageConfigurationException(
pht(
"Storage engine '%s' executed %s but did not return a valid ".
"handle ('%s') to the data: it must be nonempty and no longer ".
"than 255 characters.",
$engine_class,
'writeFile()',
$data_handle));
}
$engine_identifier = $engine->getEngineIdentifier();
if (!$engine_identifier || strlen($engine_identifier) > 32) {
throw new PhabricatorFileStorageConfigurationException(
pht(
"Storage engine '%s' returned an improper engine identifier '{%s}': ".
"it must be nonempty and no longer than 32 characters.",
$engine_class,
$engine_identifier));
}
return array($engine_identifier, $data_handle, $integrity_hash);
}
/**
* Download a remote resource over HTTP and save the response body as a file.
*
* This method respects `security.outbound-blacklist`, and protects against
* HTTP redirection (by manually following "Location" headers and verifying
* each destination). It does not protect against DNS rebinding. See
* discussion in T6755.
*/
public static function newFromFileDownload($uri, array $params = array()) {
$timeout = 5;
$redirects = array();
$current = $uri;
while (true) {
try {
if (count($redirects) > 10) {
throw new Exception(
pht('Too many redirects trying to fetch remote URI.'));
}
$resolved = PhabricatorEnv::requireValidRemoteURIForFetch(
$current,
array(
'http',
'https',
));
list($resolved_uri, $resolved_domain) = $resolved;
$current = new PhutilURI($current);
if ($current->getProtocol() == 'http') {
// For HTTP, we can use a pre-resolved URI to defuse DNS rebinding.
$fetch_uri = $resolved_uri;
$fetch_host = $resolved_domain;
} else {
// For HTTPS, we can't: cURL won't verify the SSL certificate if
// the domain has been replaced with an IP. But internal services
// presumably will not have valid certificates for rebindable
// domain names on attacker-controlled domains, so the DNS rebinding
// attack should generally not be possible anyway.
$fetch_uri = $current;
$fetch_host = null;
}
$future = id(new HTTPSFuture($fetch_uri))
->setFollowLocation(false)
->setTimeout($timeout);
if ($fetch_host !== null) {
$future->addHeader('Host', $fetch_host);
}
list($status, $body, $headers) = $future->resolve();
if ($status->isRedirect()) {
// This is an HTTP 3XX status, so look for a "Location" header.
$location = null;
foreach ($headers as $header) {
list($name, $value) = $header;
if (phutil_utf8_strtolower($name) == 'location') {
$location = $value;
break;
}
}
// HTTP 3XX status with no "Location" header, just treat this like
// a normal HTTP error.
if ($location === null) {
throw $status;
}
if (isset($redirects[$location])) {
throw new Exception(
pht('Encountered loop while following redirects.'));
}
$redirects[$location] = $location;
$current = $location;
// We'll fall off the bottom and go try this URI now.
} else if ($status->isError()) {
// This is something other than an HTTP 2XX or HTTP 3XX status, so
// just bail out.
throw $status;
} else {
// This is HTTP 2XX, so use the response body to save the file data.
// Provide a default name based on the URI, truncating it if the URI
// is exceptionally long.
$default_name = basename($uri);
$default_name = id(new PhutilUTF8StringTruncator())
->setMaximumBytes(64)
->truncateString($default_name);
$params = $params + array(
'name' => $default_name,
);
return self::newFromFileData($body, $params);
}
} catch (Exception $ex) {
if ($redirects) {
throw new PhutilProxyException(
pht(
'Failed to fetch remote URI "%s" after following %s redirect(s) '.
'(%s): %s',
$uri,
phutil_count($redirects),
implode(' > ', array_keys($redirects)),
$ex->getMessage()),
$ex);
} else {
throw $ex;
}
}
}
}
public static function normalizeFileName($file_name) {
$pattern = "@[\\x00-\\x19#%&+!~'\$\"\/=\\\\?<> ]+@";
$file_name = preg_replace($pattern, '_', $file_name);
$file_name = preg_replace('@_+@', '_', $file_name);
$file_name = trim($file_name, '_');
$disallowed_filenames = array(
'.' => 'dot',
'..' => 'dotdot',
'' => 'file',
);
$file_name = idx($disallowed_filenames, $file_name, $file_name);
return $file_name;
}
public function delete() {
// We want to delete all the rows which mark this file as the transformation
// of some other file (since we're getting rid of it). We also delete all
// the transformations of this file, so that a user who deletes an image
// doesn't need to separately hunt down and delete a bunch of thumbnails and
// resizes of it.
$outbound_xforms = id(new PhabricatorFileQuery())
->setViewer(PhabricatorUser::getOmnipotentUser())
->withTransforms(
array(
array(
'originalPHID' => $this->getPHID(),
'transform' => true,
),
))
->execute();
foreach ($outbound_xforms as $outbound_xform) {
$outbound_xform->delete();
}
$inbound_xforms = id(new PhabricatorTransformedFile())->loadAllWhere(
'transformedPHID = %s',
$this->getPHID());
$this->openTransaction();
foreach ($inbound_xforms as $inbound_xform) {
$inbound_xform->delete();
}
$ret = parent::delete();
$this->saveTransaction();
$this->deleteFileDataIfUnused(
$this->instantiateStorageEngine(),
$this->getStorageEngine(),
$this->getStorageHandle());
return $ret;
}
/**
* Destroy stored file data if there are no remaining files which reference
* it.
*/
public function deleteFileDataIfUnused(
PhabricatorFileStorageEngine $engine,
$engine_identifier,
$handle) {
// Check to see if any files are using storage.
$usage = id(new PhabricatorFile())->loadAllWhere(
'storageEngine = %s AND storageHandle = %s LIMIT 1',
$engine_identifier,
$handle);
// If there are no files using the storage, destroy the actual storage.
if (!$usage) {
try {
$engine->deleteFile($handle);
} catch (Exception $ex) {
// In the worst case, we're leaving some data stranded in a storage
// engine, which is not a big deal.
phlog($ex);
}
}
}
public static function hashFileContent($data) {
// NOTE: Hashing can fail if the algorithm isn't available in the current
// build of PHP. It's fine if we're unable to generate a content hash:
// it just means we'll store extra data when users upload duplicate files
// instead of being able to deduplicate it.
$hash = hash('sha256', $data, $raw_output = false);
if ($hash === false) {
return null;
}
return $hash;
}
public function loadFileData() {
$iterator = $this->getFileDataIterator();
return $this->loadDataFromIterator($iterator);
}
/**
* Return an iterable which emits file content bytes.
*
* @param int Offset for the start of data.
* @param int Offset for the end of data.
* @return Iterable Iterable object which emits requested data.
*/
public function getFileDataIterator($begin = null, $end = null) {
$engine = $this->instantiateStorageEngine();
$format = $this->newStorageFormat();
$iterator = $engine->getRawFileDataIterator(
$this,
$begin,
$end,
$format);
return $iterator;
}
public function getURI() {
return $this->getInfoURI();
}
public function getViewURI() {
if (!$this->getPHID()) {
throw new Exception(
pht('You must save a file before you can generate a view URI.'));
}
return $this->getCDNURI('data');
}
public function getCDNURI($request_kind) {
if (($request_kind !== 'data') &&
($request_kind !== 'download')) {
throw new Exception(
pht(
'Unknown file content request kind "%s".',
$request_kind));
}
$name = self::normalizeFileName($this->getName());
$name = phutil_escape_uri($name);
$parts = array();
$parts[] = 'file';
$parts[] = $request_kind;
// If this is an instanced install, add the instance identifier to the URI.
// Instanced configurations behind a CDN may not be able to control the
// request domain used by the CDN (as with AWS CloudFront). Embedding the
// instance identity in the path allows us to distinguish between requests
// originating from different instances but served through the same CDN.
$instance = PhabricatorEnv::getEnvConfig('cluster.instance');
- if (strlen($instance)) {
+ if (phutil_nonempty_string($instance)) {
$parts[] = '@'.$instance;
}
$parts[] = $this->getSecretKey();
$parts[] = $this->getPHID();
$parts[] = $name;
$path = '/'.implode('/', $parts);
// If this file is only partially uploaded, we're just going to return a
// local URI to make sure that Ajax works, since the page is inevitably
// going to give us an error back.
if ($this->getIsPartial()) {
return PhabricatorEnv::getURI($path);
} else {
return PhabricatorEnv::getCDNURI($path);
}
}
public function getInfoURI() {
return '/'.$this->getMonogram();
}
public function getBestURI() {
if ($this->isViewableInBrowser()) {
return $this->getViewURI();
} else {
return $this->getInfoURI();
}
}
public function getDownloadURI() {
return $this->getCDNURI('download');
}
public function getURIForTransform(PhabricatorFileTransform $transform) {
return $this->getTransformedURI($transform->getTransformKey());
}
private function getTransformedURI($transform) {
$parts = array();
$parts[] = 'file';
$parts[] = 'xform';
$instance = PhabricatorEnv::getEnvConfig('cluster.instance');
- if (strlen($instance)) {
+ if (phutil_nonempty_string($instance)) {
$parts[] = '@'.$instance;
}
$parts[] = $transform;
$parts[] = $this->getPHID();
$parts[] = $this->getSecretKey();
$path = implode('/', $parts);
$path = $path.'/';
return PhabricatorEnv::getCDNURI($path);
}
public function isViewableInBrowser() {
return ($this->getViewableMimeType() !== null);
}
public function isViewableImage() {
if (!$this->isViewableInBrowser()) {
return false;
}
$mime_map = PhabricatorEnv::getEnvConfig('files.image-mime-types');
$mime_type = $this->getMimeType();
return idx($mime_map, $mime_type);
}
public function isAudio() {
if (!$this->isViewableInBrowser()) {
return false;
}
$mime_map = PhabricatorEnv::getEnvConfig('files.audio-mime-types');
$mime_type = $this->getMimeType();
return idx($mime_map, $mime_type);
}
public function isVideo() {
if (!$this->isViewableInBrowser()) {
return false;
}
$mime_map = PhabricatorEnv::getEnvConfig('files.video-mime-types');
$mime_type = $this->getMimeType();
return idx($mime_map, $mime_type);
}
public function isPDF() {
if (!$this->isViewableInBrowser()) {
return false;
}
$mime_map = array(
'application/pdf' => 'application/pdf',
);
$mime_type = $this->getMimeType();
return idx($mime_map, $mime_type);
}
public function isTransformableImage() {
// NOTE: The way the 'gd' extension works in PHP is that you can install it
// with support for only some file types, so it might be able to handle
// PNG but not JPEG. Try to generate thumbnails for whatever we can. Setup
// warns you if you don't have complete support.
$matches = null;
$ok = preg_match(
'@^image/(gif|png|jpe?g)@',
$this->getViewableMimeType(),
$matches);
if (!$ok) {
return false;
}
switch ($matches[1]) {
case 'jpg';
case 'jpeg':
return function_exists('imagejpeg');
break;
case 'png':
return function_exists('imagepng');
break;
case 'gif':
return function_exists('imagegif');
break;
default:
throw new Exception(pht('Unknown type matched as image MIME type.'));
}
}
public static function getTransformableImageFormats() {
$supported = array();
if (function_exists('imagejpeg')) {
$supported[] = 'jpg';
}
if (function_exists('imagepng')) {
$supported[] = 'png';
}
if (function_exists('imagegif')) {
$supported[] = 'gif';
}
return $supported;
}
public function getDragAndDropDictionary() {
return array(
'id' => $this->getID(),
'phid' => $this->getPHID(),
'uri' => $this->getBestURI(),
);
}
public function instantiateStorageEngine() {
return self::buildEngine($this->getStorageEngine());
}
public static function buildEngine($engine_identifier) {
$engines = self::buildAllEngines();
foreach ($engines as $engine) {
if ($engine->getEngineIdentifier() == $engine_identifier) {
return $engine;
}
}
throw new Exception(
pht(
"Storage engine '%s' could not be located!",
$engine_identifier));
}
public static function buildAllEngines() {
return id(new PhutilClassMapQuery())
->setAncestorClass('PhabricatorFileStorageEngine')
->execute();
}
public function getViewableMimeType() {
$mime_map = PhabricatorEnv::getEnvConfig('files.viewable-mime-types');
$mime_type = $this->getMimeType();
$mime_parts = explode(';', $mime_type);
$mime_type = trim(reset($mime_parts));
return idx($mime_map, $mime_type);
}
public function getDisplayIconForMimeType() {
$mime_map = PhabricatorEnv::getEnvConfig('files.icon-mime-types');
$mime_type = $this->getMimeType();
return idx($mime_map, $mime_type, 'fa-file-o');
}
public function validateSecretKey($key) {
return ($key == $this->getSecretKey());
}
public function generateSecretKey() {
return Filesystem::readRandomCharacters(20);
}
public function setStorageProperties(array $properties) {
$this->metadata[self::METADATA_STORAGE] = $properties;
return $this;
}
public function getStorageProperties() {
return idx($this->metadata, self::METADATA_STORAGE, array());
}
public function getStorageProperty($key, $default = null) {
$properties = $this->getStorageProperties();
return idx($properties, $key, $default);
}
public function loadDataFromIterator($iterator) {
$result = '';
foreach ($iterator as $chunk) {
$result .= $chunk;
}
return $result;
}
public function updateDimensions($save = true) {
if (!$this->isViewableImage()) {
throw new Exception(pht('This file is not a viewable image.'));
}
if (!function_exists('imagecreatefromstring')) {
throw new Exception(pht('Cannot retrieve image information.'));
}
if ($this->getIsChunk()) {
throw new Exception(
pht('Refusing to assess image dimensions of file chunk.'));
}
$engine = $this->instantiateStorageEngine();
if ($engine->isChunkEngine()) {
throw new Exception(
pht('Refusing to assess image dimensions of chunked file.'));
}
$data = $this->loadFileData();
$img = @imagecreatefromstring($data);
if ($img === false) {
throw new Exception(pht('Error when decoding image.'));
}
$this->metadata[self::METADATA_IMAGE_WIDTH] = imagesx($img);
$this->metadata[self::METADATA_IMAGE_HEIGHT] = imagesy($img);
if ($save) {
$this->save();
}
return $this;
}
public function copyDimensions(PhabricatorFile $file) {
$metadata = $file->getMetadata();
$width = idx($metadata, self::METADATA_IMAGE_WIDTH);
if ($width) {
$this->metadata[self::METADATA_IMAGE_WIDTH] = $width;
}
$height = idx($metadata, self::METADATA_IMAGE_HEIGHT);
if ($height) {
$this->metadata[self::METADATA_IMAGE_HEIGHT] = $height;
}
return $this;
}
/**
* Load (or build) the {@class:PhabricatorFile} objects for builtin file
* resources. The builtin mechanism allows files shipped with Phabricator
* to be treated like normal files so that APIs do not need to special case
* things like default images or deleted files.
*
* Builtins are located in `resources/builtin/` and identified by their
* name.
*
* @param PhabricatorUser Viewing user.
* @param list<PhabricatorFilesBuiltinFile> List of builtin file specs.
* @return dict<string, PhabricatorFile> Dictionary of named builtins.
*/
public static function loadBuiltins(PhabricatorUser $user, array $builtins) {
$builtins = mpull($builtins, null, 'getBuiltinFileKey');
// NOTE: Anyone is allowed to access builtin files.
$files = id(new PhabricatorFileQuery())
->setViewer(PhabricatorUser::getOmnipotentUser())
->withBuiltinKeys(array_keys($builtins))
->execute();
$results = array();
foreach ($files as $file) {
$builtin_key = $file->getBuiltinName();
if ($builtin_key !== null) {
$results[$builtin_key] = $file;
}
}
$build = array();
foreach ($builtins as $key => $builtin) {
if (isset($results[$key])) {
continue;
}
$data = $builtin->loadBuiltinFileData();
$params = array(
'name' => $builtin->getBuiltinDisplayName(),
'canCDN' => true,
'builtin' => $key,
);
$unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
try {
$file = self::newFromFileData($data, $params);
} catch (AphrontDuplicateKeyQueryException $ex) {
$file = id(new PhabricatorFileQuery())
->setViewer(PhabricatorUser::getOmnipotentUser())
->withBuiltinKeys(array($key))
->executeOne();
if (!$file) {
throw new Exception(
pht(
'Collided mid-air when generating builtin file "%s", but '.
'then failed to load the object we collided with.',
$key));
}
}
unset($unguarded);
$file->attachObjectPHIDs(array());
$file->attachObjects(array());
$results[$key] = $file;
}
return $results;
}
/**
* Convenience wrapper for @{method:loadBuiltins}.
*
* @param PhabricatorUser Viewing user.
* @param string Single builtin name to load.
* @return PhabricatorFile Corresponding builtin file.
*/
public static function loadBuiltin(PhabricatorUser $user, $name) {
$builtin = id(new PhabricatorFilesOnDiskBuiltinFile())
->setName($name);
$key = $builtin->getBuiltinFileKey();
return idx(self::loadBuiltins($user, array($builtin)), $key);
}
public function getObjects() {
return $this->assertAttached($this->objects);
}
public function attachObjects(array $objects) {
$this->objects = $objects;
return $this;
}
public function getObjectPHIDs() {
return $this->assertAttached($this->objectPHIDs);
}
public function attachObjectPHIDs(array $object_phids) {
$this->objectPHIDs = $object_phids;
return $this;
}
public function getOriginalFile() {
return $this->assertAttached($this->originalFile);
}
public function attachOriginalFile(PhabricatorFile $file = null) {
$this->originalFile = $file;
return $this;
}
public function getImageHeight() {
if (!$this->isViewableImage()) {
return null;
}
return idx($this->metadata, self::METADATA_IMAGE_HEIGHT);
}
public function getImageWidth() {
if (!$this->isViewableImage()) {
return null;
}
return idx($this->metadata, self::METADATA_IMAGE_WIDTH);
}
public function getAltText() {
$alt = $this->getCustomAltText();
if (strlen($alt)) {
return $alt;
}
return $this->getDefaultAltText();
}
public function getCustomAltText() {
return idx($this->metadata, self::METADATA_ALT_TEXT);
}
public function setCustomAltText($value) {
$value = phutil_string_cast($value);
if (!strlen($value)) {
$value = null;
}
if ($value === null) {
unset($this->metadata[self::METADATA_ALT_TEXT]);
} else {
$this->metadata[self::METADATA_ALT_TEXT] = $value;
}
return $this;
}
public function getDefaultAltText() {
$parts = array();
$name = $this->getName();
if (strlen($name)) {
$parts[] = $name;
}
$stats = array();
$image_x = $this->getImageHeight();
$image_y = $this->getImageWidth();
if ($image_x && $image_y) {
$stats[] = pht(
"%d\xC3\x97%d px",
new PhutilNumber($image_x),
new PhutilNumber($image_y));
}
$bytes = $this->getByteSize();
if ($bytes) {
$stats[] = phutil_format_bytes($bytes);
}
if ($stats) {
$parts[] = pht('(%s)', implode(', ', $stats));
}
if (!$parts) {
return null;
}
return implode(' ', $parts);
}
public function getCanCDN() {
if (!$this->isViewableImage()) {
return false;
}
return idx($this->metadata, self::METADATA_CAN_CDN);
}
public function setCanCDN($can_cdn) {
$this->metadata[self::METADATA_CAN_CDN] = $can_cdn ? 1 : 0;
return $this;
}
public function isBuiltin() {
return ($this->getBuiltinName() !== null);
}
public function getBuiltinName() {
return idx($this->metadata, self::METADATA_BUILTIN);
}
public function setBuiltinName($name) {
$this->metadata[self::METADATA_BUILTIN] = $name;
return $this;
}
public function getIsProfileImage() {
return idx($this->metadata, self::METADATA_PROFILE);
}
public function setIsProfileImage($value) {
$this->metadata[self::METADATA_PROFILE] = $value;
return $this;
}
public function getIsChunk() {
return idx($this->metadata, self::METADATA_CHUNK);
}
public function setIsChunk($value) {
$this->metadata[self::METADATA_CHUNK] = $value;
return $this;
}
public function setIntegrityHash($integrity_hash) {
$this->metadata[self::METADATA_INTEGRITY] = $integrity_hash;
return $this;
}
public function getIntegrityHash() {
return idx($this->metadata, self::METADATA_INTEGRITY);
}
public function newIntegrityHash() {
$engine = $this->instantiateStorageEngine();
if ($engine->isChunkEngine()) {
return null;
}
$format = $this->newStorageFormat();
$storage_handle = $this->getStorageHandle();
$data = $engine->readFile($storage_handle);
return $engine->newIntegrityHash($data, $format);
}
/**
* Write the policy edge between this file and some object.
*
* @param phid Object PHID to attach to.
* @return this
*/
public function attachToObject($phid) {
$attachment_table = new PhabricatorFileAttachment();
$attachment_conn = $attachment_table->establishConnection('w');
queryfx(
$attachment_conn,
'INSERT INTO %R (objectPHID, filePHID, attachmentMode,
attacherPHID, dateCreated, dateModified)
VALUES (%s, %s, %s, %ns, %d, %d)
ON DUPLICATE KEY UPDATE
attachmentMode = VALUES(attachmentMode),
attacherPHID = VALUES(attacherPHID),
dateModified = VALUES(dateModified)',
$attachment_table,
$phid,
$this->getPHID(),
PhabricatorFileAttachment::MODE_ATTACH,
null,
PhabricatorTime::getNow(),
PhabricatorTime::getNow());
return $this;
}
/**
* Configure a newly created file object according to specified parameters.
*
* This method is called both when creating a file from fresh data, and
* when creating a new file which reuses existing storage.
*
* @param map<string, wild> Bag of parameters, see @{class:PhabricatorFile}
* for documentation.
* @return this
*/
private function readPropertiesFromParameters(array $params) {
PhutilTypeSpec::checkMap(
$params,
array(
'name' => 'optional string',
'authorPHID' => 'optional string',
'ttl.relative' => 'optional int',
'ttl.absolute' => 'optional int',
'viewPolicy' => 'optional string',
'isExplicitUpload' => 'optional bool',
'canCDN' => 'optional bool',
'profile' => 'optional bool',
'format' => 'optional string|PhabricatorFileStorageFormat',
'mime-type' => 'optional string',
'builtin' => 'optional string',
'storageEngines' => 'optional list<PhabricatorFileStorageEngine>',
'chunk' => 'optional bool',
));
$file_name = idx($params, 'name');
$this->setName($file_name);
$author_phid = idx($params, 'authorPHID');
$this->setAuthorPHID($author_phid);
$absolute_ttl = idx($params, 'ttl.absolute');
$relative_ttl = idx($params, 'ttl.relative');
if ($absolute_ttl !== null && $relative_ttl !== null) {
throw new Exception(
pht(
'Specify an absolute TTL or a relative TTL, but not both.'));
} else if ($absolute_ttl !== null) {
if ($absolute_ttl < PhabricatorTime::getNow()) {
throw new Exception(
pht(
'Absolute TTL must be in the present or future, but TTL "%s" '.
'is in the past.',
$absolute_ttl));
}
$this->setTtl($absolute_ttl);
} else if ($relative_ttl !== null) {
if ($relative_ttl < 0) {
throw new Exception(
pht(
'Relative TTL must be zero or more seconds, but "%s" is '.
'negative.',
$relative_ttl));
}
$max_relative = phutil_units('365 days in seconds');
if ($relative_ttl > $max_relative) {
throw new Exception(
pht(
'Relative TTL must not be more than "%s" seconds, but TTL '.
'"%s" was specified.',
$max_relative,
$relative_ttl));
}
$absolute_ttl = PhabricatorTime::getNow() + $relative_ttl;
$this->setTtl($absolute_ttl);
}
$view_policy = idx($params, 'viewPolicy');
if ($view_policy) {
$this->setViewPolicy($params['viewPolicy']);
}
$is_explicit = (idx($params, 'isExplicitUpload') ? 1 : 0);
$this->setIsExplicitUpload($is_explicit);
$can_cdn = idx($params, 'canCDN');
if ($can_cdn) {
$this->setCanCDN(true);
}
$builtin = idx($params, 'builtin');
if ($builtin) {
$this->setBuiltinName($builtin);
$this->setBuiltinKey($builtin);
}
$profile = idx($params, 'profile');
if ($profile) {
$this->setIsProfileImage(true);
}
$mime_type = idx($params, 'mime-type');
if ($mime_type) {
$this->setMimeType($mime_type);
}
$is_chunk = idx($params, 'chunk');
if ($is_chunk) {
$this->setIsChunk(true);
}
return $this;
}
public function getRedirectResponse() {
$uri = $this->getBestURI();
// TODO: This is a bit iffy. Sometimes, getBestURI() returns a CDN URI
// (if the file is a viewable image) and sometimes a local URI (if not).
// For now, just detect which one we got and configure the response
// appropriately. In the long run, if this endpoint is served from a CDN
// domain, we can't issue a local redirect to an info URI (which is not
// present on the CDN domain). We probably never actually issue local
// redirects here anyway, since we only ever transform viewable images
// right now.
$is_external = strlen(id(new PhutilURI($uri))->getDomain());
return id(new AphrontRedirectResponse())
->setIsExternal($is_external)
->setURI($uri);
}
public function newDownloadResponse() {
// We're cheating a little bit here and relying on the fact that
// getDownloadURI() always returns a fully qualified URI with a complete
// domain.
return id(new AphrontRedirectResponse())
->setIsExternal(true)
->setCloseDialogBeforeRedirect(true)
->setURI($this->getDownloadURI());
}
public function attachTransforms(array $map) {
$this->transforms = $map;
return $this;
}
public function getTransform($key) {
return $this->assertAttachedKey($this->transforms, $key);
}
public function newStorageFormat() {
$key = $this->getStorageFormat();
$template = PhabricatorFileStorageFormat::requireFormat($key);
$format = id(clone $template)
->setFile($this);
return $format;
}
/* -( PhabricatorApplicationTransactionInterface )------------------------- */
public function getApplicationTransactionEditor() {
return new PhabricatorFileEditor();
}
public function getApplicationTransactionTemplate() {
return new PhabricatorFileTransaction();
}
/* -( PhabricatorPolicyInterface Implementation )-------------------------- */
public function getCapabilities() {
return array(
PhabricatorPolicyCapability::CAN_VIEW,
PhabricatorPolicyCapability::CAN_EDIT,
);
}
public function getPolicy($capability) {
switch ($capability) {
case PhabricatorPolicyCapability::CAN_VIEW:
if ($this->isBuiltin()) {
return PhabricatorPolicies::getMostOpenPolicy();
}
if ($this->getIsProfileImage()) {
return PhabricatorPolicies::getMostOpenPolicy();
}
return $this->getViewPolicy();
case PhabricatorPolicyCapability::CAN_EDIT:
return PhabricatorPolicies::POLICY_NOONE;
}
}
public function hasAutomaticCapability($capability, PhabricatorUser $viewer) {
$viewer_phid = $viewer->getPHID();
if ($viewer_phid) {
if ($this->getAuthorPHID() == $viewer_phid) {
return true;
}
}
switch ($capability) {
case PhabricatorPolicyCapability::CAN_VIEW:
// If you can see the file this file is a transform of, you can see
// this file.
if ($this->getOriginalFile()) {
return true;
}
// If you can see any object this file is attached to, you can see
// the file.
return (count($this->getObjects()) > 0);
}
return false;
}
public function describeAutomaticCapability($capability) {
$out = array();
$out[] = pht('The user who uploaded a file can always view and edit it.');
switch ($capability) {
case PhabricatorPolicyCapability::CAN_VIEW:
$out[] = pht(
'Files attached to objects are visible to users who can view '.
'those objects.');
$out[] = pht(
'Thumbnails are visible only to users who can view the original '.
'file.');
break;
}
return $out;
}
/* -( PhabricatorSubscribableInterface Implementation )-------------------- */
public function isAutomaticallySubscribed($phid) {
return ($this->authorPHID == $phid);
}
/* -( PhabricatorTokenReceiverInterface )---------------------------------- */
public function getUsersToNotifyOfTokenGiven() {
return array(
$this->getAuthorPHID(),
);
}
/* -( PhabricatorDestructibleInterface )----------------------------------- */
public function destroyObjectPermanently(
PhabricatorDestructionEngine $engine) {
$this->openTransaction();
$this->delete();
$this->saveTransaction();
}
/* -( PhabricatorConduitResultInterface )---------------------------------- */
public function getFieldSpecificationsForConduit() {
return array(
id(new PhabricatorConduitSearchFieldSpecification())
->setKey('name')
->setType('string')
->setDescription(pht('The name of the file.')),
id(new PhabricatorConduitSearchFieldSpecification())
->setKey('uri')
->setType('uri')
->setDescription(pht('View URI for the file.')),
id(new PhabricatorConduitSearchFieldSpecification())
->setKey('dataURI')
->setType('uri')
->setDescription(pht('Download URI for the file data.')),
id(new PhabricatorConduitSearchFieldSpecification())
->setKey('size')
->setType('int')
->setDescription(pht('File size, in bytes.')),
);
}
public function getFieldValuesForConduit() {
return array(
'name' => $this->getName(),
'uri' => PhabricatorEnv::getURI($this->getURI()),
'dataURI' => $this->getCDNURI('data'),
'size' => (int)$this->getByteSize(),
'alt' => array(
'custom' => $this->getCustomAltText(),
'default' => $this->getDefaultAltText(),
),
);
}
public function getConduitSearchAttachments() {
return array();
}
/* -( PhabricatorNgramInterface )------------------------------------------ */
public function newNgrams() {
return array(
id(new PhabricatorFileNameNgrams())
->setValue($this->getName()),
);
}
}
diff --git a/src/applications/metamta/engine/PhabricatorMailEmailEngine.php b/src/applications/metamta/engine/PhabricatorMailEmailEngine.php
index 6c9cf1b356..4278aa96a1 100644
--- a/src/applications/metamta/engine/PhabricatorMailEmailEngine.php
+++ b/src/applications/metamta/engine/PhabricatorMailEmailEngine.php
@@ -1,654 +1,654 @@
<?php
final class PhabricatorMailEmailEngine
extends PhabricatorMailMessageEngine {
public function newMessage() {
$mailer = $this->getMailer();
$mail = $this->getMail();
$message = new PhabricatorMailEmailMessage();
$from_address = $this->newFromEmailAddress();
$message->setFromAddress($from_address);
$reply_address = $this->newReplyToEmailAddress();
if ($reply_address) {
$message->setReplyToAddress($reply_address);
}
$to_addresses = $this->newToEmailAddresses();
$cc_addresses = $this->newCCEmailAddresses();
if (!$to_addresses && !$cc_addresses) {
$mail->setMessage(
pht(
'Message has no valid recipients: all To/CC are disabled, '.
'invalid, or configured not to receive this mail.'));
return null;
}
// If this email describes a mail processing error, we rate limit outbound
// messages to each individual address. This prevents messes where
// something is stuck in a loop or dumps a ton of messages on us suddenly.
if ($mail->getIsErrorEmail()) {
$all_recipients = array();
foreach ($to_addresses as $to_address) {
$all_recipients[] = $to_address->getAddress();
}
foreach ($cc_addresses as $cc_address) {
$all_recipients[] = $cc_address->getAddress();
}
if ($this->shouldRateLimitMail($all_recipients)) {
$mail->setMessage(
pht(
'This is an error email, but one or more recipients have '.
'exceeded the error email rate limit. Declining to deliver '.
'message.'));
return null;
}
}
// Some mailers require a valid "To:" in order to deliver mail. If we
// don't have any "To:", try to fill it in with a placeholder "To:".
// If that also fails, move the "Cc:" line to "To:".
if (!$to_addresses) {
$void_address = $this->newVoidEmailAddress();
$to_addresses = array($void_address);
}
$to_addresses = $this->getUniqueEmailAddresses($to_addresses);
$cc_addresses = $this->getUniqueEmailAddresses(
$cc_addresses,
$to_addresses);
$message->setToAddresses($to_addresses);
$message->setCCAddresses($cc_addresses);
$attachments = $this->newEmailAttachments();
$message->setAttachments($attachments);
$subject = $this->newEmailSubject();
$message->setSubject($subject);
$headers = $this->newEmailHeaders();
foreach ($this->newEmailThreadingHeaders($mailer) as $threading_header) {
$headers[] = $threading_header;
}
$stamps = $mail->getMailStamps();
if ($stamps) {
$headers[] = $this->newEmailHeader(
'X-Phabricator-Stamps',
implode(' ', $stamps));
}
$must_encrypt = $mail->getMustEncrypt();
$raw_body = $mail->getBody();
$body = $raw_body;
if ($must_encrypt) {
$parts = array();
$encrypt_uri = $mail->getMustEncryptURI();
if (!strlen($encrypt_uri)) {
$encrypt_phid = $mail->getRelatedPHID();
if ($encrypt_phid) {
$encrypt_uri = urisprintf(
'/object/%s/',
$encrypt_phid);
}
}
if (strlen($encrypt_uri)) {
$parts[] = pht(
'This secure message is notifying you of a change to this object:');
$parts[] = PhabricatorEnv::getProductionURI($encrypt_uri);
}
$parts[] = pht(
'The content for this message can only be transmitted over a '.
'secure channel. To view the message content, follow this '.
'link:');
$parts[] = PhabricatorEnv::getProductionURI($mail->getURI());
$body = implode("\n\n", $parts);
} else {
$body = $raw_body;
}
$body_limit = PhabricatorEnv::getEnvConfig('metamta.email-body-limit');
$body = phutil_string_cast($body);
if (strlen($body) > $body_limit) {
$body = id(new PhutilUTF8StringTruncator())
->setMaximumBytes($body_limit)
->truncateString($body);
$body .= "\n";
$body .= pht('(This email was truncated at %d bytes.)', $body_limit);
}
$message->setTextBody($body);
$body_limit -= strlen($body);
// If we sent a different message body than we were asked to, record
// what we actually sent to make debugging and diagnostics easier.
if ($body !== $raw_body) {
$mail->setDeliveredBody($body);
}
if ($must_encrypt) {
$send_html = false;
} else {
$send_html = $this->shouldSendHTML();
}
if ($send_html) {
$html_body = $mail->getHTMLBody();
if (phutil_nonempty_string($html_body)) {
// NOTE: We just drop the entire HTML body if it won't fit. Safely
// truncating HTML is hard, and we already have the text body to fall
// back to.
if (strlen($html_body) <= $body_limit) {
$message->setHTMLBody($html_body);
$body_limit -= strlen($html_body);
}
}
}
// Pass the headers to the mailer, then save the state so we can show
// them in the web UI. If the mail must be encrypted, we remove headers
// which are not on a strict whitelist to avoid disclosing information.
$filtered_headers = $this->filterHeaders($headers, $must_encrypt);
$message->setHeaders($filtered_headers);
$mail->setUnfilteredHeaders($headers);
$mail->setDeliveredHeaders($headers);
if (PhabricatorEnv::getEnvConfig('phabricator.silent')) {
$mail->setMessage(
pht(
'This software is running in silent mode. See `%s` '.
'in the configuration to change this setting.',
'phabricator.silent'));
return null;
}
return $message;
}
/* -( Message Components )------------------------------------------------- */
private function newFromEmailAddress() {
$from_address = $this->newDefaultEmailAddress();
$mail = $this->getMail();
// If the mail content must be encrypted, always disguise the sender.
$must_encrypt = $mail->getMustEncrypt();
if ($must_encrypt) {
return $from_address;
}
// If we have a raw "From" address, use that.
$raw_from = $mail->getRawFrom();
if ($raw_from) {
list($from_email, $from_name) = $raw_from;
return $this->newEmailAddress($from_email, $from_name);
}
// Otherwise, use as much of the information for any sending entity as
// we can.
$from_phid = $mail->getFrom();
$actor = $this->getActor($from_phid);
if ($actor) {
$actor_email = $actor->getEmailAddress();
$actor_name = $actor->getName();
} else {
$actor_email = null;
$actor_name = null;
}
$send_as_user = PhabricatorEnv::getEnvConfig('metamta.can-send-as-user');
if ($send_as_user) {
if ($actor_email !== null) {
$from_address->setAddress($actor_email);
}
}
if ($actor_name !== null) {
$from_address->setDisplayName($actor_name);
}
return $from_address;
}
private function newReplyToEmailAddress() {
$mail = $this->getMail();
$reply_raw = $mail->getReplyTo();
if (!phutil_nonempty_string($reply_raw)) {
return null;
}
$reply_address = new PhutilEmailAddress($reply_raw);
// If we have a sending object, change the display name.
$from_phid = $mail->getFrom();
$actor = $this->getActor($from_phid);
if ($actor) {
$reply_address->setDisplayName($actor->getName());
}
// If we don't have a display name, fill in a default.
if (!strlen($reply_address->getDisplayName())) {
$reply_address->setDisplayName(PlatformSymbols::getPlatformServerName());
}
return $reply_address;
}
private function newToEmailAddresses() {
$mail = $this->getMail();
$phids = $mail->getToPHIDs();
$addresses = $this->newEmailAddressesFromActorPHIDs($phids);
foreach ($mail->getRawToAddresses() as $raw_address) {
$addresses[] = new PhutilEmailAddress($raw_address);
}
return $addresses;
}
private function newCCEmailAddresses() {
$mail = $this->getMail();
$phids = $mail->getCcPHIDs();
return $this->newEmailAddressesFromActorPHIDs($phids);
}
private function newEmailAddressesFromActorPHIDs(array $phids) {
$mail = $this->getMail();
$phids = $mail->expandRecipients($phids);
$addresses = array();
foreach ($phids as $phid) {
$actor = $this->getActor($phid);
if (!$actor) {
continue;
}
if (!$actor->isDeliverable()) {
continue;
}
$addresses[] = new PhutilEmailAddress($actor->getEmailAddress());
}
return $addresses;
}
private function newEmailSubject() {
$mail = $this->getMail();
$is_threaded = (bool)$mail->getThreadID();
$must_encrypt = $mail->getMustEncrypt();
$subject = array();
if ($is_threaded) {
if ($this->shouldAddRePrefix()) {
$subject[] = 'Re:';
}
}
$subject_prefix = $mail->getSubjectPrefix();
$subject_prefix = phutil_string_cast($subject_prefix);
$subject_prefix = trim($subject_prefix);
$subject[] = $subject_prefix;
// If mail content must be encrypted, we replace the subject with
// a generic one.
if ($must_encrypt) {
$encrypt_subject = $mail->getMustEncryptSubject();
if (!strlen($encrypt_subject)) {
$encrypt_subject = pht('Object Updated');
}
$subject[] = $encrypt_subject;
} else {
$vary_prefix = $mail->getVarySubjectPrefix();
if (phutil_nonempty_string($vary_prefix)) {
if ($this->shouldVarySubject()) {
$subject[] = $vary_prefix;
}
}
$subject[] = $mail->getSubject();
}
foreach ($subject as $key => $part) {
if (!phutil_nonempty_string($part)) {
unset($subject[$key]);
}
}
$subject = implode(' ', $subject);
return $subject;
}
private function newEmailHeaders() {
$mail = $this->getMail();
$headers = array();
$headers[] = $this->newEmailHeader(
'X-Phabricator-Sent-This-Message',
'Yes');
$headers[] = $this->newEmailHeader(
'X-Mail-Transport-Agent',
'MetaMTA');
// Some clients respect this to suppress OOF and other auto-responses.
$headers[] = $this->newEmailHeader(
'X-Auto-Response-Suppress',
'All');
$mailtags = $mail->getMailTags();
if ($mailtags) {
$tag_header = array();
foreach ($mailtags as $mailtag) {
$tag_header[] = '<'.$mailtag.'>';
}
$tag_header = implode(', ', $tag_header);
$headers[] = $this->newEmailHeader(
'X-Phabricator-Mail-Tags',
$tag_header);
}
$value = $mail->getHeaders();
foreach ($value as $pair) {
list($header_key, $header_value) = $pair;
// NOTE: If we have \n in a header, SES rejects the email.
$header_value = str_replace("\n", ' ', $header_value);
$headers[] = $this->newEmailHeader($header_key, $header_value);
}
$is_bulk = $mail->getIsBulk();
if ($is_bulk) {
$headers[] = $this->newEmailHeader('Precedence', 'bulk');
}
if ($mail->getMustEncrypt()) {
$headers[] = $this->newEmailHeader('X-Phabricator-Must-Encrypt', 'Yes');
}
$related_phid = $mail->getRelatedPHID();
if ($related_phid) {
$headers[] = $this->newEmailHeader('Thread-Topic', $related_phid);
}
$headers[] = $this->newEmailHeader(
'X-Phabricator-Mail-ID',
$mail->getID());
$unique = Filesystem::readRandomCharacters(16);
$headers[] = $this->newEmailHeader(
'X-Phabricator-Send-Attempt',
$unique);
return $headers;
}
private function newEmailThreadingHeaders() {
$mailer = $this->getMailer();
$mail = $this->getMail();
$headers = array();
$thread_id = $mail->getThreadID();
if (!phutil_nonempty_string($thread_id)) {
return $headers;
}
$is_first = $mail->getIsFirstMessage();
// NOTE: Gmail freaks out about In-Reply-To and References which aren't in
// the form "<string@domain.tld>"; this is also required by RFC 2822,
// although some clients are more liberal in what they accept.
$domain = $this->newMailDomain();
$thread_id = '<'.$thread_id.'@'.$domain.'>';
if ($is_first && $mailer->supportsMessageIDHeader()) {
$headers[] = $this->newEmailHeader('Message-ID', $thread_id);
} else {
$in_reply_to = $thread_id;
$references = array($thread_id);
$parent_id = $mail->getParentMessageID();
if ($parent_id) {
$in_reply_to = $parent_id;
// By RFC 2822, the most immediate parent should appear last
// in the "References" header, so this order is intentional.
$references[] = $parent_id;
}
$references = implode(' ', $references);
$headers[] = $this->newEmailHeader('In-Reply-To', $in_reply_to);
$headers[] = $this->newEmailHeader('References', $references);
}
$thread_index = $this->generateThreadIndex($thread_id, $is_first);
$headers[] = $this->newEmailHeader('Thread-Index', $thread_index);
return $headers;
}
private function newEmailAttachments() {
$mail = $this->getMail();
// If the mail content must be encrypted, don't add attachments.
$must_encrypt = $mail->getMustEncrypt();
if ($must_encrypt) {
return array();
}
return $mail->getAttachments();
}
/* -( Preferences )-------------------------------------------------------- */
private function shouldAddRePrefix() {
$preferences = $this->getPreferences();
$value = $preferences->getSettingValue(
PhabricatorEmailRePrefixSetting::SETTINGKEY);
return ($value == PhabricatorEmailRePrefixSetting::VALUE_RE_PREFIX);
}
private function shouldVarySubject() {
$preferences = $this->getPreferences();
$value = $preferences->getSettingValue(
PhabricatorEmailVarySubjectsSetting::SETTINGKEY);
return ($value == PhabricatorEmailVarySubjectsSetting::VALUE_VARY_SUBJECTS);
}
private function shouldSendHTML() {
$preferences = $this->getPreferences();
$value = $preferences->getSettingValue(
PhabricatorEmailFormatSetting::SETTINGKEY);
return ($value == PhabricatorEmailFormatSetting::VALUE_HTML_EMAIL);
}
/* -( Utilities )---------------------------------------------------------- */
private function newEmailHeader($name, $value) {
return id(new PhabricatorMailHeader())
->setName($name)
->setValue($value);
}
private function newEmailAddress($address, $name = null) {
$object = id(new PhutilEmailAddress())
->setAddress($address);
if (strlen($name)) {
$object->setDisplayName($name);
}
return $object;
}
public function newDefaultEmailAddress() {
$raw_address = PhabricatorEnv::getEnvConfig('metamta.default-address');
- if (!strlen($raw_address)) {
+ if (!$raw_address) {
$domain = $this->newMailDomain();
$raw_address = "noreply@{$domain}";
}
$address = new PhutilEmailAddress($raw_address);
if (!phutil_nonempty_string($address->getDisplayName())) {
$address->setDisplayName(PlatformSymbols::getPlatformServerName());
}
return $address;
}
public function newVoidEmailAddress() {
return $this->newDefaultEmailAddress();
}
private function newMailDomain() {
$domain = PhabricatorEnv::getEnvConfig('metamta.reply-handler-domain');
- if (strlen($domain)) {
+ if ($domain) {
return $domain;
}
$install_uri = PhabricatorEnv::getURI('/');
$install_uri = new PhutilURI($install_uri);
return $install_uri->getDomain();
}
private function filterHeaders(array $headers, $must_encrypt) {
assert_instances_of($headers, 'PhabricatorMailHeader');
if (!$must_encrypt) {
return $headers;
}
$whitelist = array(
'In-Reply-To',
'Message-ID',
'Precedence',
'References',
'Thread-Index',
'Thread-Topic',
'X-Mail-Transport-Agent',
'X-Auto-Response-Suppress',
'X-Phabricator-Sent-This-Message',
'X-Phabricator-Must-Encrypt',
'X-Phabricator-Mail-ID',
'X-Phabricator-Send-Attempt',
);
// NOTE: The major header we want to drop is "X-Phabricator-Mail-Tags".
// This header contains a significant amount of meaningful information
// about the object.
$whitelist_map = array();
foreach ($whitelist as $term) {
$whitelist_map[phutil_utf8_strtolower($term)] = true;
}
foreach ($headers as $key => $header) {
$name = $header->getName();
$name = phutil_utf8_strtolower($name);
if (!isset($whitelist_map[$name])) {
unset($headers[$key]);
}
}
return $headers;
}
private function getUniqueEmailAddresses(
array $addresses,
array $exclude = array()) {
assert_instances_of($addresses, 'PhutilEmailAddress');
assert_instances_of($exclude, 'PhutilEmailAddress');
$seen = array();
foreach ($exclude as $address) {
$seen[$address->getAddress()] = true;
}
foreach ($addresses as $key => $address) {
$raw_address = $address->getAddress();
if (isset($seen[$raw_address])) {
unset($addresses[$key]);
continue;
}
$seen[$raw_address] = true;
}
return array_values($addresses);
}
private function generateThreadIndex($seed, $is_first_mail) {
// When threading, Outlook ignores the 'References' and 'In-Reply-To'
// headers that most clients use. Instead, it uses a custom 'Thread-Index'
// header. The format of this header is something like this (from
// camel-exchange-folder.c in Evolution Exchange):
/* A new post to a folder gets a 27-byte-long thread index. (The value
* is apparently unique but meaningless.) Each reply to a post gets a
* 32-byte-long thread index whose first 27 bytes are the same as the
* parent's thread index. Each reply to any of those gets a
* 37-byte-long thread index, etc. The Thread-Index header contains a
* base64 representation of this value.
*/
// The specific implementation uses a 27-byte header for the first email
// a recipient receives, and a random 5-byte suffix (32 bytes total)
// thereafter. This means that all the replies are (incorrectly) siblings,
// but it would be very difficult to keep track of the entire tree and this
// gets us reasonable client behavior.
$base = substr(md5($seed), 0, 27);
if (!$is_first_mail) {
// Not totally sure, but it seems like outlook orders replies by
// thread-index rather than timestamp, so to get these to show up in the
// right order we use the time as the last 4 bytes.
$base .= ' '.pack('N', time());
}
return base64_encode($base);
}
private function shouldRateLimitMail(array $all_recipients) {
try {
PhabricatorSystemActionEngine::willTakeAction(
$all_recipients,
new PhabricatorMetaMTAErrorMailAction(),
1);
return false;
} catch (PhabricatorSystemActionRateLimitException $ex) {
return true;
}
}
}
diff --git a/src/applications/notification/client/PhabricatorNotificationServerRef.php b/src/applications/notification/client/PhabricatorNotificationServerRef.php
index 714ad5f5d7..c5b0f2f8aa 100644
--- a/src/applications/notification/client/PhabricatorNotificationServerRef.php
+++ b/src/applications/notification/client/PhabricatorNotificationServerRef.php
@@ -1,255 +1,255 @@
<?php
final class PhabricatorNotificationServerRef
extends Phobject {
private $type;
private $host;
private $port;
private $protocol;
private $path;
private $isDisabled;
const KEY_REFS = 'notification.refs';
public function setType($type) {
$this->type = $type;
return $this;
}
public function getType() {
return $this->type;
}
public function setHost($host) {
$this->host = $host;
return $this;
}
public function getHost() {
return $this->host;
}
public function setPort($port) {
$this->port = $port;
return $this;
}
public function getPort() {
return $this->port;
}
public function setProtocol($protocol) {
$this->protocol = $protocol;
return $this;
}
public function getProtocol() {
return $this->protocol;
}
public function setPath($path) {
$this->path = $path;
return $this;
}
public function getPath() {
return $this->path;
}
public function setIsDisabled($is_disabled) {
$this->isDisabled = $is_disabled;
return $this;
}
public function getIsDisabled() {
return $this->isDisabled;
}
public static function getLiveServers() {
$cache = PhabricatorCaches::getRequestCache();
$refs = $cache->getKey(self::KEY_REFS);
if (!$refs) {
$refs = self::newRefs();
$cache->setKey(self::KEY_REFS, $refs);
}
return $refs;
}
public static function newRefs() {
$configs = PhabricatorEnv::getEnvConfig('notification.servers');
$refs = array();
foreach ($configs as $config) {
$ref = id(new self())
->setType($config['type'])
->setHost($config['host'])
->setPort($config['port'])
->setProtocol($config['protocol'])
->setPath(idx($config, 'path'))
->setIsDisabled(idx($config, 'disabled', false));
$refs[] = $ref;
}
return $refs;
}
public static function getEnabledServers() {
$servers = self::getLiveServers();
foreach ($servers as $key => $server) {
if ($server->getIsDisabled()) {
unset($servers[$key]);
}
}
return array_values($servers);
}
public static function getEnabledAdminServers() {
$servers = self::getEnabledServers();
foreach ($servers as $key => $server) {
if (!$server->isAdminServer()) {
unset($servers[$key]);
}
}
return array_values($servers);
}
public static function getEnabledClientServers($with_protocol) {
$servers = self::getEnabledServers();
foreach ($servers as $key => $server) {
if ($server->isAdminServer()) {
unset($servers[$key]);
continue;
}
$protocol = $server->getProtocol();
if ($protocol != $with_protocol) {
unset($servers[$key]);
continue;
}
}
return array_values($servers);
}
public function isAdminServer() {
return ($this->type == 'admin');
}
public function getURI($to_path = null) {
$full_path = rtrim($this->getPath(), '/').'/'.ltrim($to_path, '/');
$uri = id(new PhutilURI('http://'.$this->getHost()))
->setProtocol($this->getProtocol())
->setPort($this->getPort())
->setPath($full_path);
$instance = PhabricatorEnv::getEnvConfig('cluster.instance');
- if (strlen($instance)) {
+ if (phutil_nonempty_string($instance)) {
$uri->replaceQueryParam('instance', $instance);
}
return $uri;
}
public function getWebsocketURI($to_path = null) {
$instance = PhabricatorEnv::getEnvConfig('cluster.instance');
if (strlen($instance)) {
$to_path = $to_path.'~'.$instance.'/';
}
$uri = $this->getURI($to_path);
if ($this->getProtocol() == 'https') {
$uri->setProtocol('wss');
} else {
$uri->setProtocol('ws');
}
return $uri;
}
public function testClient() {
if ($this->isAdminServer()) {
throw new Exception(
pht('Unable to test client on an admin server!'));
}
$server_uri = $this->getURI();
try {
id(new HTTPSFuture($server_uri))
->setTimeout(2)
->resolvex();
} catch (HTTPFutureHTTPResponseStatus $ex) {
// This is what we expect when things are working correctly.
if ($ex->getStatusCode() == 501) {
return true;
}
throw $ex;
}
throw new Exception(
pht('Got HTTP 200, but expected HTTP 501 (WebSocket Upgrade)!'));
}
public function loadServerStatus() {
if (!$this->isAdminServer()) {
throw new Exception(
pht(
'Unable to load server status: this is not an admin server!'));
}
$server_uri = $this->getURI('/status/');
list($body) = $this->newFuture($server_uri)
->resolvex();
return phutil_json_decode($body);
}
public function postMessage(array $data) {
if (!$this->isAdminServer()) {
throw new Exception(
pht('Unable to post message: this is not an admin server!'));
}
$server_uri = $this->getURI('/');
$payload = phutil_json_encode($data);
$this->newFuture($server_uri, $payload)
->setMethod('POST')
->resolvex();
}
private function newFuture($uri, $data = null) {
if ($data === null) {
$future = new HTTPSFuture($uri);
} else {
$future = new HTTPSFuture($uri, $data);
}
$future->setTimeout(2);
// At one point, a HackerOne researcher reported a "Location:" redirect
// attack here (if the attacker can gain control of the notification
// server or the configuration).
// Although this attack is not particularly concerning, we don't expect
// Aphlict to ever issue a "Location:" header, so receiving one indicates
// something is wrong and declining to follow the header may make debugging
// easier.
$future->setFollowLocation(false);
return $future;
}
}
diff --git a/src/applications/repository/engine/PhabricatorRepositoryPullEngine.php b/src/applications/repository/engine/PhabricatorRepositoryPullEngine.php
index 12aee9bc51..5bcf4554fd 100644
--- a/src/applications/repository/engine/PhabricatorRepositoryPullEngine.php
+++ b/src/applications/repository/engine/PhabricatorRepositoryPullEngine.php
@@ -1,844 +1,844 @@
<?php
/**
* Manages execution of `git pull` and `hg pull` commands for
* @{class:PhabricatorRepository} objects. Used by
* @{class:PhabricatorRepositoryPullLocalDaemon}.
*
* This class also covers initial working copy setup through `git clone`,
* `git init`, `hg clone`, `hg init`, or `svnadmin create`.
*
* @task pull Pulling Working Copies
* @task git Pulling Git Working Copies
* @task hg Pulling Mercurial Working Copies
* @task svn Pulling Subversion Working Copies
* @task internal Internals
*/
final class PhabricatorRepositoryPullEngine
extends PhabricatorRepositoryEngine {
/* -( Pulling Working Copies )--------------------------------------------- */
public function pullRepository() {
$repository = $this->getRepository();
$lock = $this->newRepositoryLock($repository, 'repo.pull', true);
try {
$lock->lock();
} catch (PhutilLockException $ex) {
throw new DiffusionDaemonLockException(
pht(
'Another process is currently updating repository "%s", '.
'skipping pull.',
$repository->getDisplayName()));
}
try {
$result = $this->pullRepositoryWithLock();
} catch (Exception $ex) {
$lock->unlock();
throw $ex;
}
$lock->unlock();
return $result;
}
private function pullRepositoryWithLock() {
$repository = $this->getRepository();
$viewer = PhabricatorUser::getOmnipotentUser();
if ($repository->isReadOnly()) {
$this->skipPull(
pht(
"Skipping pull on read-only repository.\n\n%s",
$repository->getReadOnlyMessageForDisplay()));
}
$is_hg = false;
$is_git = false;
$is_svn = false;
$vcs = $repository->getVersionControlSystem();
switch ($vcs) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
// We never pull a local copy of non-hosted Subversion repositories.
if (!$repository->isHosted()) {
$this->skipPull(
pht(
'Repository "%s" is a non-hosted Subversion repository, which '.
'does not require a local working copy to be pulled.',
$repository->getDisplayName()));
return;
}
$is_svn = true;
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
$is_git = true;
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
$is_hg = true;
break;
default:
$this->abortPull(pht('Unknown VCS "%s"!', $vcs));
break;
}
$local_path = $repository->getLocalPath();
if ($local_path === null) {
$this->abortPull(
pht(
'No local path is configured for repository "%s".',
$repository->getDisplayName()));
}
try {
$dirname = dirname($local_path);
if (!Filesystem::pathExists($dirname)) {
Filesystem::createDirectory($dirname, 0755, $recursive = true);
}
if (!Filesystem::pathExists($local_path)) {
$this->logPull(
pht(
'Creating a new working copy for repository "%s".',
$repository->getDisplayName()));
if ($is_git) {
$this->executeGitCreate();
} else if ($is_hg) {
$this->executeMercurialCreate();
} else {
$this->executeSubversionCreate();
}
}
id(new DiffusionRepositoryClusterEngine())
->setViewer($viewer)
->setRepository($repository)
->synchronizeWorkingCopyBeforeRead();
if (!$repository->isHosted()) {
$this->logPull(
pht(
'Updating the working copy for repository "%s".',
$repository->getDisplayName()));
if ($is_git) {
$this->executeGitUpdate();
} else if ($is_hg) {
$this->executeMercurialUpdate();
}
}
if ($repository->isHosted()) {
if ($is_git) {
$this->installGitHook();
} else if ($is_svn) {
$this->installSubversionHook();
} else if ($is_hg) {
$this->installMercurialHook();
}
foreach ($repository->getHookDirectories() as $directory) {
$this->installHookDirectory($directory);
}
}
if ($is_git) {
$this->updateGitWorkingCopyConfiguration();
}
} catch (Exception $ex) {
$this->abortPull(
pht(
"Pull of '%s' failed: %s",
$repository->getDisplayName(),
$ex->getMessage()),
$ex);
}
$this->donePull();
return $this;
}
private function skipPull($message) {
$this->log($message);
$this->donePull();
}
private function abortPull($message, Exception $ex = null) {
$code_error = PhabricatorRepositoryStatusMessage::CODE_ERROR;
$this->updateRepositoryInitStatus($code_error, $message);
if ($ex) {
throw $ex;
} else {
throw new Exception($message);
}
}
private function logPull($message) {
$this->log($message);
}
private function donePull() {
$code_okay = PhabricatorRepositoryStatusMessage::CODE_OKAY;
$this->updateRepositoryInitStatus($code_okay);
}
private function updateRepositoryInitStatus($code, $message = null) {
$this->getRepository()->writeStatusMessage(
PhabricatorRepositoryStatusMessage::TYPE_INIT,
$code,
array(
'message' => $message,
));
}
private function installHook($path, array $hook_argv = array()) {
$this->log(pht('Installing commit hook to "%s"...', $path));
$repository = $this->getRepository();
$identifier = $this->getHookContextIdentifier($repository);
$root = dirname(phutil_get_library_root('phabricator'));
$bin = $root.'/bin/commit-hook';
$full_php_path = Filesystem::resolveBinary('php');
$cmd = csprintf(
'exec %s -f %s -- %s %Ls "$@"',
$full_php_path,
$bin,
$identifier,
$hook_argv);
$hook = "#!/bin/sh\nexport TERM=dumb\n{$cmd}\n";
Filesystem::writeFile($path, $hook);
Filesystem::changePermissions($path, 0755);
}
private function installHookDirectory($path) {
$readme = pht(
"To add custom hook scripts to this repository, add them to this ".
"directory.\n\nPhabricator will run any executables in this directory ".
"after running its own checks, as though they were normal hook ".
"scripts.");
Filesystem::createDirectory($path, 0755);
Filesystem::writeFile($path.'/README', $readme);
}
private function getHookContextIdentifier(PhabricatorRepository $repository) {
$identifier = $repository->getPHID();
$instance = PhabricatorEnv::getEnvConfig('cluster.instance');
- if (strlen($instance)) {
+ if (phutil_nonempty_string($instance)) {
$identifier = "{$identifier}:{$instance}";
}
return $identifier;
}
/* -( Pulling Git Working Copies )----------------------------------------- */
/**
* @task git
*/
private function executeGitCreate() {
$repository = $this->getRepository();
$path = rtrim($repository->getLocalPath(), '/');
// See T13448. In all cases, we create repositories by using "git init"
// to build a bare, empty working copy. If we try to use "git clone"
// instead, we'll pull in too many refs if "Fetch Refs" is also
// configured. There's no apparent way to make "git clone" behave narrowly
// and no apparent reason to bother.
$repository->execxRemoteCommand(
'init --bare -- %s',
$path);
}
/**
* @task git
*/
private function executeGitUpdate() {
$repository = $this->getRepository();
// See T13479. We previously used "--show-toplevel", but this stopped
// working in Git 2.25.0 when run in a bare repository.
// NOTE: As of Git 2.21.1, "git rev-parse" can not parse "--" in its
// argument list, so we can not specify arguments unambiguously. Any
// version of Git which does not recognize the "--git-dir" flag will
// treat this as a request to parse the literal refname "--git-dir".
list($err, $stdout) = $repository->execLocalCommand(
'rev-parse --git-dir');
$repository_root = null;
$path = $repository->getLocalPath();
if (!$err) {
$repository_root = Filesystem::resolvePath(
rtrim($stdout, "\n"),
$path);
// If we're in a bare Git repository, the "--git-dir" will be the
// root directory. If we're in a working copy, the "--git-dir" will
// be the ".git/" directory.
// Test if the result is the root directory. If it is, we're in good
// shape and appear to be inside a bare repository. If not, take the
// parent directory to get out of the ".git/" folder.
if (!Filesystem::pathsAreEquivalent($repository_root, $path)) {
$repository_root = dirname($repository_root);
}
}
$message = null;
if ($err) {
// Try to raise a more tailored error message in the more common case
// of the user creating an empty directory. (We could try to remove it,
// but might not be able to, and it's much simpler to raise a good
// message than try to navigate those waters.)
if (is_dir($path)) {
$files = Filesystem::listDirectory($path, $include_hidden = true);
if (!$files) {
$message = pht(
'Expected to find a Git repository at "%s", but there is an '.
'empty directory there. Remove the directory. A daemon will '.
'construct the working copy for you.',
$path);
} else {
$message = pht(
'Expected to find a Git repository at "%s", but there is '.
'a non-repository directory (with other stuff in it) there. '.
'Move or remove this directory. A daemon will construct '.
'the working copy for you.',
$path);
}
} else if (is_file($path)) {
$message = pht(
'Expected to find a Git repository at "%s", but there is a '.
'file there instead. Move or remove this file. A daemon will '.
'construct the working copy for you.',
$path);
} else {
$message = pht(
'Expected to find a git repository at "%s", but did not.',
$path);
}
} else {
// Prior to Git 2.25.0, we used "--show-toplevel", which had a weird
// case here when the working copy was inside another working copy.
// The switch to "--git-dir" seems to have resolved this; we now seem
// to find the nearest git directory and thus the correct repository
// root.
if (!Filesystem::pathsAreEquivalent($repository_root, $path)) {
$err = true;
$message = pht(
'Expected to find a Git repository at "%s", but the actual Git '.
'repository root for this directory is "%s". Something is '.
'misconfigured. This directory should be writable by the daemons '.
'and not inside another Git repository.',
$path,
$repository_root);
}
}
if ($err && $repository->canDestroyWorkingCopy()) {
phlog(
pht(
"Repository working copy at '%s' failed sanity check; ".
"destroying and re-cloning. %s",
$path,
$message));
Filesystem::remove($path);
$this->executeGitCreate();
} else if ($err) {
throw new Exception($message);
}
// Load the refs we're planning to fetch from the remote repository.
$remote_refs = $this->loadGitRemoteRefs(
$repository,
$repository->getRemoteURIEnvelope(),
$is_local = false);
// Load the refs we're planning to fetch from the local repository, by
// using the local working copy path as the "remote" repository URI.
$local_refs = $this->loadGitRemoteRefs(
$repository,
new PhutilOpaqueEnvelope($path),
$is_local = true);
// See T13448. The "git fetch --prune ..." flag only prunes local refs
// matching the refspecs we pass it. If "Fetch Refs" is configured, we'll
// pass it a very narrow list of refspecs, and it won't prune older refs
// that aren't currently subject to fetching.
// Since we want to prune everything that isn't (a) on the fetch list and
// (b) in the remote, handle pruning of any surplus leftover refs ourselves
// before we fetch anything.
// (We don't have to do this if "Fetch Refs" isn't set up, since "--prune"
// will work in that case, but it's a little simpler to always go down the
// same code path.)
$surplus_refs = array();
foreach ($local_refs as $local_ref => $local_hash) {
$remote_hash = idx($remote_refs, $local_ref);
if ($remote_hash === null) {
$surplus_refs[] = $local_ref;
}
}
if ($surplus_refs) {
$this->log(
pht(
'Found %s surplus local ref(s) to delete.',
phutil_count($surplus_refs)));
foreach ($surplus_refs as $surplus_ref) {
$this->log(
pht(
'Deleting surplus local ref "%s" ("%s").',
$surplus_ref,
$local_refs[$surplus_ref]));
$repository->execLocalCommand(
'update-ref -d %R --',
$surplus_ref);
unset($local_refs[$surplus_ref]);
}
}
if ($remote_refs === $local_refs) {
$this->log(
pht(
'Skipping fetch because local and remote refs are already '.
'identical.'));
return false;
}
$this->logRefDifferences($remote_refs, $local_refs);
$fetch_rules = $this->getGitFetchRules($repository);
// For very old non-bare working copies, we need to use "--update-head-ok"
// to tell Git that it is allowed to overwrite whatever is currently
// checked out. See T13280.
$future = $repository->getRemoteCommandFuture(
'fetch --no-tags --update-head-ok -- %P %Ls',
$repository->getRemoteURIEnvelope(),
$fetch_rules);
$future
->setCWD($path)
->resolvex();
}
private function getGitRefRules(PhabricatorRepository $repository) {
$ref_rules = $repository->getFetchRules($repository);
if (!$ref_rules) {
$ref_rules = array(
'refs/*',
);
}
return $ref_rules;
}
private function getGitFetchRules(PhabricatorRepository $repository) {
$ref_rules = $this->getGitRefRules($repository);
// Rewrite each ref rule "X" into "+X:X".
// The "X" means "fetch ref X".
// The "...:X" means "...and copy it into local ref X".
// The "+..." means "...and overwrite the local ref if it already exists".
$fetch_rules = array();
foreach ($ref_rules as $key => $ref_rule) {
$fetch_rules[] = sprintf(
'+%s:%s',
$ref_rule,
$ref_rule);
}
return $fetch_rules;
}
/**
* @task git
*/
private function installGitHook() {
$repository = $this->getRepository();
$root = $repository->getLocalPath();
if ($repository->isWorkingCopyBare()) {
$path = '/hooks/pre-receive';
} else {
$path = '/.git/hooks/pre-receive';
}
$this->installHook($root.$path);
}
private function updateGitWorkingCopyConfiguration() {
$repository = $this->getRepository();
// See T5963. When you "git clone" from a remote with no "master", the
// client warns you that it isn't sure what it should check out as an
// initial state:
// warning: remote HEAD refers to nonexistent ref, unable to checkout
// We can tell the client what it should check out by making "HEAD"
// point somewhere. However:
//
// (1) If we don't set "receive.denyDeleteCurrent" to "ignore" and a user
// tries to delete the default branch, Git raises an error and refuses.
// We want to allow this; we already have sufficient protections around
// dangerous changes and do not need to special case the default branch.
//
// (2) A repository may have a nonexistent default branch configured.
// For now, we just respect configuration. This will raise a warning when
// users clone the repository.
//
// In any case, these changes are both advisory, so ignore any errors we
// may encounter.
// We do this for both hosted and observed repositories. Although it is
// not terribly common to clone from Phabricator's copy of an observed
// repository, it works fine and makes sense occasionally.
if ($repository->isWorkingCopyBare()) {
$repository->execLocalCommand(
'config -- receive.denyDeleteCurrent ignore');
$repository->execLocalCommand(
'symbolic-ref HEAD %s',
'refs/heads/'.$repository->getDefaultBranch());
}
}
private function loadGitRemoteRefs(
PhabricatorRepository $repository,
PhutilOpaqueEnvelope $remote_envelope,
$is_local) {
// See T13448. When listing local remotes, we want to list everything,
// not just refs we expect to fetch. This allows us to detect that we have
// undesirable refs (which have been deleted in the remote, but are still
// present locally) so we can update our state to reflect the correct
// remote state.
if ($is_local) {
$ref_rules = array();
} else {
$ref_rules = $this->getGitRefRules($repository);
// NOTE: "git ls-remote" does not support "--" until circa January 2016.
// See T12416. None of the flags to "ls-remote" appear dangerous, but
// refuse to list any refs beginning with "-" just in case.
foreach ($ref_rules as $ref_rule) {
if (preg_match('/^-/', $ref_rule)) {
throw new Exception(
pht(
'Refusing to list potentially dangerous ref ("%s") beginning '.
'with "-".',
$ref_rule));
}
}
}
list($stdout) = $repository->execxRemoteCommand(
'ls-remote %P %Ls',
$remote_envelope,
$ref_rules);
// Empty repositories don't have any refs.
if (!strlen(rtrim($stdout))) {
return array();
}
$map = array();
$lines = phutil_split_lines($stdout, false);
foreach ($lines as $line) {
list($hash, $name) = preg_split('/\s+/', $line, 2);
// If the remote has a HEAD, just ignore it.
if ($name == 'HEAD') {
continue;
}
// If the remote ref is itself a remote ref, ignore it.
if (preg_match('(^refs/remotes/)', $name)) {
continue;
}
$map[$name] = $hash;
}
ksort($map);
return $map;
}
private function loadGitLocalRefs(PhabricatorRepository $repository) {
$refs = id(new DiffusionLowLevelGitRefQuery())
->setRepository($repository)
->execute();
$map = array();
foreach ($refs as $ref) {
$fields = $ref->getRawFields();
$map[idx($fields, 'refname')] = $ref->getCommitIdentifier();
}
ksort($map);
return $map;
}
private function logRefDifferences(array $remote, array $local) {
$all = $local + $remote;
$differences = array();
foreach ($all as $key => $ignored) {
$remote_ref = idx($remote, $key, pht('<null>'));
$local_ref = idx($local, $key, pht('<null>'));
if ($remote_ref !== $local_ref) {
$differences[] = pht(
'%s (remote: "%s", local: "%s")',
$key,
$remote_ref,
$local_ref);
}
}
$this->log(
pht(
"Updating repository after detecting ref differences:\n%s",
implode("\n", $differences)));
}
/* -( Pulling Mercurial Working Copies )----------------------------------- */
/**
* @task hg
*/
private function executeMercurialCreate() {
$repository = $this->getRepository();
$path = rtrim($repository->getLocalPath(), '/');
if ($repository->isHosted()) {
$repository->execxRemoteCommand(
'init -- %s',
$path);
} else {
$remote = $repository->getRemoteURIEnvelope();
// NOTE: Mercurial prior to 3.2.4 has an severe command injection
// vulnerability. See: <http://bit.ly/19B58E9>
// On vulnerable versions of Mercurial, we refuse to clone remotes which
// contain characters which may be interpreted by the shell.
$hg_binary = PhutilBinaryAnalyzer::getForBinary('hg');
$is_vulnerable = $hg_binary->isMercurialVulnerableToInjection();
if ($is_vulnerable) {
$cleartext = $remote->openEnvelope();
// The use of "%R" here is an attempt to limit collateral damage
// for normal URIs because it isn't clear how long this vulnerability
// has been around for.
$escaped = csprintf('%R', $cleartext);
if ((string)$escaped !== (string)$cleartext) {
throw new Exception(
pht(
'You have an old version of Mercurial (%s) which has a severe '.
'command injection security vulnerability. The remote URI for '.
'this repository (%s) is potentially unsafe. Upgrade Mercurial '.
'to at least 3.2.4 to clone it.',
$hg_binary->getBinaryVersion(),
$repository->getMonogram()));
}
}
try {
$repository->execxRemoteCommand(
'clone --noupdate -- %P %s',
$remote,
$path);
} catch (Exception $ex) {
$message = $ex->getMessage();
$message = $this->censorMercurialErrorMessage($message);
throw new Exception($message);
}
}
}
/**
* @task hg
*/
private function executeMercurialUpdate() {
$repository = $this->getRepository();
$path = $repository->getLocalPath();
// This is a local command, but needs credentials.
$remote = $repository->getRemoteURIEnvelope();
$future = $repository->getRemoteCommandFuture('pull -- %P', $remote);
$future->setCWD($path);
try {
$future->resolvex();
} catch (CommandException $ex) {
$err = $ex->getError();
$stdout = $ex->getStdout();
// NOTE: Between versions 2.1 and 2.1.1, Mercurial changed the behavior
// of "hg pull" to return 1 in case of a successful pull with no changes.
// This behavior has been reverted, but users who updated between Feb 1,
// 2012 and Mar 1, 2012 will have the erroring version. Do a dumb test
// against stdout to check for this possibility.
// NOTE: Mercurial has translated versions, which translate this error
// string. In a translated version, the string will be something else,
// like "aucun changement trouve". There didn't seem to be an easy way
// to handle this (there are hard ways but this is not a common problem
// and only creates log spam, not application failures). Assume English.
// TODO: Remove this once we're far enough in the future that deployment
// of 2.1 is exceedingly rare?
if ($err == 1 && preg_match('/no changes found/', $stdout)) {
return;
} else {
$message = $ex->getMessage();
$message = $this->censorMercurialErrorMessage($message);
throw new Exception($message);
}
}
}
/**
* Censor response bodies from Mercurial error messages.
*
* When Mercurial attempts to clone an HTTP repository but does not
* receive a response it expects, it emits the response body in the
* command output.
*
* This represents a potential SSRF issue, because an attacker with
* permission to create repositories can create one which points at the
* remote URI for some local service, then read the response from the
* error message. To prevent this, censor response bodies out of error
* messages.
*
* @param string Uncensored Mercurial command output.
* @return string Censored Mercurial command output.
*/
private function censorMercurialErrorMessage($message) {
return preg_replace(
'/^---%<---.*/sm',
pht('<Response body omitted from Mercurial error message.>')."\n",
$message);
}
/**
* @task hg
*/
private function installMercurialHook() {
$repository = $this->getRepository();
$path = $repository->getLocalPath().'/.hg/hgrc';
$identifier = $this->getHookContextIdentifier($repository);
$root = dirname(phutil_get_library_root('phabricator'));
$bin = $root.'/bin/commit-hook';
$data = array();
$data[] = '[hooks]';
// This hook handles normal pushes.
$data[] = csprintf(
'pretxnchangegroup.phabricator = TERM=dumb %s %s %s',
$bin,
$identifier,
'pretxnchangegroup');
// This one handles creating bookmarks.
$data[] = csprintf(
'prepushkey.phabricator = TERM=dumb %s %s %s',
$bin,
$identifier,
'prepushkey');
$data[] = null;
$data = implode("\n", $data);
$this->log('%s', pht('Installing commit hook config to "%s"...', $path));
Filesystem::writeFile($path, $data);
}
/* -( Pulling Subversion Working Copies )---------------------------------- */
/**
* @task svn
*/
private function executeSubversionCreate() {
$repository = $this->getRepository();
$path = rtrim($repository->getLocalPath(), '/');
execx('svnadmin create -- %s', $path);
}
/**
* @task svn
*/
private function installSubversionHook() {
$repository = $this->getRepository();
$root = $repository->getLocalPath();
$path = '/hooks/pre-commit';
$this->installHook($root.$path);
$revprop_path = '/hooks/pre-revprop-change';
$revprop_argv = array(
'--hook-mode',
'svn-revprop',
);
$this->installHook($root.$revprop_path, $revprop_argv);
}
}
diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php
index c68eb22668..5c7c9ff33d 100644
--- a/src/applications/repository/storage/PhabricatorRepository.php
+++ b/src/applications/repository/storage/PhabricatorRepository.php
@@ -1,2884 +1,2885 @@
<?php
/**
* @task uri Repository URI Management
* @task publishing Publishing
* @task sync Cluster Synchronization
*/
final class PhabricatorRepository extends PhabricatorRepositoryDAO
implements
PhabricatorApplicationTransactionInterface,
PhabricatorPolicyInterface,
PhabricatorFlaggableInterface,
PhabricatorMarkupInterface,
PhabricatorDestructibleInterface,
PhabricatorDestructibleCodexInterface,
PhabricatorProjectInterface,
PhabricatorSpacesInterface,
PhabricatorConduitResultInterface,
PhabricatorFulltextInterface,
PhabricatorFerretInterface {
/**
* Shortest hash we'll recognize in raw "a829f32" form.
*/
const MINIMUM_UNQUALIFIED_HASH = 7;
/**
* Shortest hash we'll recognize in qualified "rXab7ef2f8" form.
*/
const MINIMUM_QUALIFIED_HASH = 5;
/**
* Minimum number of commits to an empty repository to trigger "import" mode.
*/
const IMPORT_THRESHOLD = 7;
const LOWPRI_THRESHOLD = 64;
const TABLE_PATH = 'repository_path';
const TABLE_PATHCHANGE = 'repository_pathchange';
const TABLE_FILESYSTEM = 'repository_filesystem';
const TABLE_SUMMARY = 'repository_summary';
const TABLE_LINTMESSAGE = 'repository_lintmessage';
const TABLE_PARENTS = 'repository_parents';
const TABLE_COVERAGE = 'repository_coverage';
const STATUS_ACTIVE = 'active';
const STATUS_INACTIVE = 'inactive';
protected $name;
protected $callsign;
protected $repositorySlug;
protected $uuid;
protected $viewPolicy;
protected $editPolicy;
protected $pushPolicy;
protected $profileImagePHID;
protected $versionControlSystem;
protected $details = array();
protected $credentialPHID;
protected $almanacServicePHID;
protected $spacePHID;
protected $localPath;
private $commitCount = self::ATTACHABLE;
private $mostRecentCommit = self::ATTACHABLE;
private $projectPHIDs = self::ATTACHABLE;
private $uris = self::ATTACHABLE;
private $profileImageFile = self::ATTACHABLE;
public static function initializeNewRepository(PhabricatorUser $actor) {
$app = id(new PhabricatorApplicationQuery())
->setViewer($actor)
->withClasses(array('PhabricatorDiffusionApplication'))
->executeOne();
$view_policy = $app->getPolicy(DiffusionDefaultViewCapability::CAPABILITY);
$edit_policy = $app->getPolicy(DiffusionDefaultEditCapability::CAPABILITY);
$push_policy = $app->getPolicy(DiffusionDefaultPushCapability::CAPABILITY);
$repository = id(new PhabricatorRepository())
->setViewPolicy($view_policy)
->setEditPolicy($edit_policy)
->setPushPolicy($push_policy)
->setSpacePHID($actor->getDefaultSpacePHID());
// Put the repository in "Importing" mode until we finish
// parsing it.
$repository->setDetail('importing', true);
return $repository;
}
protected function getConfiguration() {
return array(
self::CONFIG_AUX_PHID => true,
self::CONFIG_SERIALIZATION => array(
'details' => self::SERIALIZATION_JSON,
),
self::CONFIG_COLUMN_SCHEMA => array(
'name' => 'sort255',
'callsign' => 'sort32?',
'repositorySlug' => 'sort64?',
'versionControlSystem' => 'text32',
'uuid' => 'text64?',
'pushPolicy' => 'policy',
'credentialPHID' => 'phid?',
'almanacServicePHID' => 'phid?',
'localPath' => 'text128?',
'profileImagePHID' => 'phid?',
),
self::CONFIG_KEY_SCHEMA => array(
'callsign' => array(
'columns' => array('callsign'),
'unique' => true,
),
'key_name' => array(
'columns' => array('name(128)'),
),
'key_vcs' => array(
'columns' => array('versionControlSystem'),
),
'key_slug' => array(
'columns' => array('repositorySlug'),
'unique' => true,
),
'key_local' => array(
'columns' => array('localPath'),
'unique' => true,
),
),
) + parent::getConfiguration();
}
public function generatePHID() {
return PhabricatorPHID::generateNewPHID(
PhabricatorRepositoryRepositoryPHIDType::TYPECONST);
}
public static function getStatusMap() {
return array(
self::STATUS_ACTIVE => array(
'name' => pht('Active'),
'isTracked' => 1,
),
self::STATUS_INACTIVE => array(
'name' => pht('Inactive'),
'isTracked' => 0,
),
);
}
public static function getStatusNameMap() {
return ipull(self::getStatusMap(), 'name');
}
public function getStatus() {
if ($this->isTracked()) {
return self::STATUS_ACTIVE;
} else {
return self::STATUS_INACTIVE;
}
}
public function toDictionary() {
return array(
'id' => $this->getID(),
'name' => $this->getName(),
'phid' => $this->getPHID(),
'callsign' => $this->getCallsign(),
'monogram' => $this->getMonogram(),
'vcs' => $this->getVersionControlSystem(),
'uri' => PhabricatorEnv::getProductionURI($this->getURI()),
'remoteURI' => (string)$this->getRemoteURI(),
'description' => $this->getDetail('description'),
'isActive' => $this->isTracked(),
'isHosted' => $this->isHosted(),
'isImporting' => $this->isImporting(),
'encoding' => $this->getDefaultTextEncoding(),
'staging' => array(
'supported' => $this->supportsStaging(),
'prefix' => 'phabricator',
'uri' => $this->getStagingURI(),
),
);
}
public function getDefaultTextEncoding() {
return $this->getDetail('encoding', 'UTF-8');
}
public function getMonogram() {
$callsign = $this->getCallsign();
if (phutil_nonempty_string($callsign)) {
return "r{$callsign}";
}
$id = $this->getID();
return "R{$id}";
}
public function getDisplayName() {
$slug = $this->getRepositorySlug();
if (phutil_nonempty_string($slug)) {
return $slug;
}
return $this->getMonogram();
}
public function getAllMonograms() {
$monograms = array();
$monograms[] = 'R'.$this->getID();
$callsign = $this->getCallsign();
if (strlen($callsign)) {
$monograms[] = 'r'.$callsign;
}
return $monograms;
}
public function setLocalPath($path) {
// Convert any extra slashes ("//") in the path to a single slash ("/").
$path = preg_replace('(//+)', '/', $path);
return parent::setLocalPath($path);
}
public function getDetail($key, $default = null) {
return idx($this->details, $key, $default);
}
public function setDetail($key, $value) {
$this->details[$key] = $value;
return $this;
}
public function attachCommitCount($count) {
$this->commitCount = $count;
return $this;
}
public function getCommitCount() {
return $this->assertAttached($this->commitCount);
}
public function attachMostRecentCommit(
PhabricatorRepositoryCommit $commit = null) {
$this->mostRecentCommit = $commit;
return $this;
}
public function getMostRecentCommit() {
return $this->assertAttached($this->mostRecentCommit);
}
public function getDiffusionBrowseURIForPath(
PhabricatorUser $user,
$path,
$line = null,
$branch = null) {
$drequest = DiffusionRequest::newFromDictionary(
array(
'user' => $user,
'repository' => $this,
'path' => $path,
'branch' => $branch,
));
return $drequest->generateURI(
array(
'action' => 'browse',
'line' => $line,
));
}
public function getSubversionBaseURI($commit = null) {
$subpath = $this->getDetail('svn-subpath');
if (!phutil_nonempty_string($subpath)) {
$subpath = null;
}
return $this->getSubversionPathURI($subpath, $commit);
}
public function getSubversionPathURI($path = null, $commit = null) {
$vcs = $this->getVersionControlSystem();
if ($vcs != PhabricatorRepositoryType::REPOSITORY_TYPE_SVN) {
throw new Exception(pht('Not a subversion repository!'));
}
if ($this->isHosted()) {
$uri = 'file://'.$this->getLocalPath();
} else {
$uri = $this->getDetail('remote-uri');
}
$uri = rtrim($uri, '/');
if (phutil_nonempty_string($path)) {
$path = rawurlencode($path);
$path = str_replace('%2F', '/', $path);
$uri = $uri.'/'.ltrim($path, '/');
}
if ($path !== null || $commit !== null) {
$uri .= '@';
}
if ($commit !== null) {
$uri .= $commit;
}
return $uri;
}
public function attachProjectPHIDs(array $project_phids) {
$this->projectPHIDs = $project_phids;
return $this;
}
public function getProjectPHIDs() {
return $this->assertAttached($this->projectPHIDs);
}
/**
* Get the name of the directory this repository should clone or checkout
* into. For example, if the repository name is "Example Repository", a
* reasonable name might be "example-repository". This is used to help users
* get reasonable results when cloning repositories, since they generally do
* not want to clone into directories called "X/" or "Example Repository/".
*
* @return string
*/
public function getCloneName() {
$name = $this->getRepositorySlug();
// Make some reasonable effort to produce reasonable default directory
// names from repository names.
if (!strlen($name)) {
$name = $this->getName();
$name = phutil_utf8_strtolower($name);
$name = preg_replace('@[ -/:->]+@', '-', $name);
$name = trim($name, '-');
if (!strlen($name)) {
$name = $this->getCallsign();
}
}
return $name;
}
public static function isValidRepositorySlug($slug) {
try {
self::assertValidRepositorySlug($slug);
return true;
} catch (Exception $ex) {
return false;
}
}
public static function assertValidRepositorySlug($slug) {
if (!strlen($slug)) {
throw new Exception(
pht(
'The empty string is not a valid repository short name. '.
'Repository short names must be at least one character long.'));
}
if (strlen($slug) > 64) {
throw new Exception(
pht(
'The name "%s" is not a valid repository short name. Repository '.
'short names must not be longer than 64 characters.',
$slug));
}
if (preg_match('/[^a-zA-Z0-9._-]/', $slug)) {
throw new Exception(
pht(
'The name "%s" is not a valid repository short name. Repository '.
'short names may only contain letters, numbers, periods, hyphens '.
'and underscores.',
$slug));
}
if (!preg_match('/^[a-zA-Z0-9]/', $slug)) {
throw new Exception(
pht(
'The name "%s" is not a valid repository short name. Repository '.
'short names must begin with a letter or number.',
$slug));
}
if (!preg_match('/[a-zA-Z0-9]\z/', $slug)) {
throw new Exception(
pht(
'The name "%s" is not a valid repository short name. Repository '.
'short names must end with a letter or number.',
$slug));
}
if (preg_match('/__|--|\\.\\./', $slug)) {
throw new Exception(
pht(
'The name "%s" is not a valid repository short name. Repository '.
'short names must not contain multiple consecutive underscores, '.
'hyphens, or periods.',
$slug));
}
if (preg_match('/^[A-Z]+\z/', $slug)) {
throw new Exception(
pht(
'The name "%s" is not a valid repository short name. Repository '.
'short names may not contain only uppercase letters.',
$slug));
}
if (preg_match('/^\d+\z/', $slug)) {
throw new Exception(
pht(
'The name "%s" is not a valid repository short name. Repository '.
'short names may not contain only numbers.',
$slug));
}
if (preg_match('/\\.git/', $slug)) {
throw new Exception(
pht(
'The name "%s" is not a valid repository short name. Repository '.
'short names must not end in ".git". This suffix will be added '.
'automatically in appropriate contexts.',
$slug));
}
}
public static function assertValidCallsign($callsign) {
if (!strlen($callsign)) {
throw new Exception(
pht(
'A repository callsign must be at least one character long.'));
}
if (strlen($callsign) > 32) {
throw new Exception(
pht(
'The callsign "%s" is not a valid repository callsign. Callsigns '.
'must be no more than 32 bytes long.',
$callsign));
}
if (!preg_match('/^[A-Z]+\z/', $callsign)) {
throw new Exception(
pht(
'The callsign "%s" is not a valid repository callsign. Callsigns '.
'may only contain UPPERCASE letters.',
$callsign));
}
}
public function getProfileImageURI() {
return $this->getProfileImageFile()->getBestURI();
}
public function attachProfileImageFile(PhabricatorFile $file) {
$this->profileImageFile = $file;
return $this;
}
public function getProfileImageFile() {
return $this->assertAttached($this->profileImageFile);
}
/* -( Remote Command Execution )------------------------------------------- */
public function execRemoteCommand($pattern /* , $arg, ... */) {
$args = func_get_args();
return $this->newRemoteCommandFuture($args)->resolve();
}
public function execxRemoteCommand($pattern /* , $arg, ... */) {
$args = func_get_args();
return $this->newRemoteCommandFuture($args)->resolvex();
}
public function getRemoteCommandFuture($pattern /* , $arg, ... */) {
$args = func_get_args();
return $this->newRemoteCommandFuture($args);
}
public function passthruRemoteCommand($pattern /* , $arg, ... */) {
$args = func_get_args();
return $this->newRemoteCommandPassthru($args)->resolve();
}
private function newRemoteCommandFuture(array $argv) {
return $this->newRemoteCommandEngine($argv)
->newFuture();
}
private function newRemoteCommandPassthru(array $argv) {
return $this->newRemoteCommandEngine($argv)
->setPassthru(true)
->newFuture();
}
private function newRemoteCommandEngine(array $argv) {
return DiffusionCommandEngine::newCommandEngine($this)
->setArgv($argv)
->setCredentialPHID($this->getCredentialPHID())
->setURI($this->getRemoteURIObject());
}
/* -( Local Command Execution )-------------------------------------------- */
public function execLocalCommand($pattern /* , $arg, ... */) {
$args = func_get_args();
return $this->newLocalCommandFuture($args)->resolve();
}
public function execxLocalCommand($pattern /* , $arg, ... */) {
$args = func_get_args();
return $this->newLocalCommandFuture($args)->resolvex();
}
public function getLocalCommandFuture($pattern /* , $arg, ... */) {
$args = func_get_args();
return $this->newLocalCommandFuture($args);
}
public function passthruLocalCommand($pattern /* , $arg, ... */) {
$args = func_get_args();
return $this->newLocalCommandPassthru($args)->resolve();
}
private function newLocalCommandFuture(array $argv) {
$this->assertLocalExists();
$future = DiffusionCommandEngine::newCommandEngine($this)
->setArgv($argv)
->newFuture();
if ($this->usesLocalWorkingCopy()) {
$future->setCWD($this->getLocalPath());
}
return $future;
}
private function newLocalCommandPassthru(array $argv) {
$this->assertLocalExists();
$future = DiffusionCommandEngine::newCommandEngine($this)
->setArgv($argv)
->setPassthru(true)
->newFuture();
if ($this->usesLocalWorkingCopy()) {
$future->setCWD($this->getLocalPath());
}
return $future;
}
public function getURI() {
$short_name = $this->getRepositorySlug();
if (phutil_nonempty_string($short_name)) {
return "/source/{$short_name}/";
}
$callsign = $this->getCallsign();
if (phutil_nonempty_string($callsign)) {
return "/diffusion/{$callsign}/";
}
$id = $this->getID();
return "/diffusion/{$id}/";
}
public function getPathURI($path) {
return $this->getURI().ltrim($path, '/');
}
public function getCommitURI($identifier) {
$callsign = $this->getCallsign();
if (phutil_nonempty_string($callsign)) {
return "/r{$callsign}{$identifier}";
}
$id = $this->getID();
return "/R{$id}:{$identifier}";
}
public static function parseRepositoryServicePath($request_path, $vcs) {
$is_git = ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_GIT);
$patterns = array(
'(^'.
'(?P<base>/?(?:diffusion|source)/(?P<identifier>[^/]+))'.
'(?P<path>.*)'.
'\z)',
);
$identifier = null;
foreach ($patterns as $pattern) {
$matches = null;
if (!preg_match($pattern, $request_path, $matches)) {
continue;
}
$identifier = $matches['identifier'];
if ($is_git) {
$identifier = preg_replace('/\\.git\z/', '', $identifier);
}
$base = $matches['base'];
$path = $matches['path'];
break;
}
if ($identifier === null) {
return null;
}
return array(
'identifier' => $identifier,
'base' => $base,
'path' => $path,
);
}
public function getCanonicalPath($request_path) {
$standard_pattern =
'(^'.
'(?P<prefix>/(?:diffusion|source)/)'.
'(?P<identifier>[^/]+)'.
'(?P<suffix>(?:/.*)?)'.
'\z)';
$matches = null;
if (preg_match($standard_pattern, $request_path, $matches)) {
$suffix = $matches['suffix'];
return $this->getPathURI($suffix);
}
$commit_pattern =
'(^'.
'(?P<prefix>/)'.
'(?P<monogram>'.
'(?:'.
'r(?P<repositoryCallsign>[A-Z]+)'.
'|'.
'R(?P<repositoryID>[1-9]\d*):'.
')'.
'(?P<commit>[a-f0-9]+)'.
')'.
'\z)';
$matches = null;
if (preg_match($commit_pattern, $request_path, $matches)) {
$commit = $matches['commit'];
return $this->getCommitURI($commit);
}
return null;
}
public function generateURI(array $params) {
$req_branch = false;
$req_commit = false;
$action = idx($params, 'action');
switch ($action) {
case 'history':
case 'clone':
case 'blame':
case 'browse':
case 'document':
case 'change':
case 'lastmodified':
case 'tags':
case 'branches':
case 'lint':
case 'pathtree':
case 'refs':
case 'compare':
break;
case 'branch':
// NOTE: This does not actually require a branch, and won't have one
// in Subversion. Possibly this should be more clear.
break;
case 'commit':
case 'rendering-ref':
$req_commit = true;
break;
default:
throw new Exception(
pht(
'Action "%s" is not a valid repository URI action.',
$action));
}
$path = idx($params, 'path');
$branch = idx($params, 'branch');
$commit = idx($params, 'commit');
$line = idx($params, 'line');
$head = idx($params, 'head');
$against = idx($params, 'against');
if ($req_commit && !strlen($commit)) {
throw new Exception(
pht(
'Diffusion URI action "%s" requires commit!',
$action));
}
if ($req_branch && !strlen($branch)) {
throw new Exception(
pht(
'Diffusion URI action "%s" requires branch!',
$action));
}
if ($action === 'commit') {
return $this->getCommitURI($commit);
}
if (phutil_nonempty_string($path)) {
$path = ltrim($path, '/');
$path = str_replace(array(';', '$'), array(';;', '$$'), $path);
$path = phutil_escape_uri($path);
}
$raw_branch = $branch;
if (phutil_nonempty_string($branch)) {
$branch = phutil_escape_uri_path_component($branch);
$path = "{$branch}/{$path}";
}
$raw_commit = $commit;
if (phutil_nonempty_scalar($commit)) {
$commit = str_replace('$', '$$', $commit);
$commit = ';'.phutil_escape_uri($commit);
}
$line = phutil_string_cast($line);
if (phutil_nonempty_string($line)) {
$line = '$'.phutil_escape_uri($line);
}
$query = array();
switch ($action) {
case 'change':
case 'history':
case 'blame':
case 'browse':
case 'document':
case 'lastmodified':
case 'tags':
case 'branches':
case 'lint':
case 'pathtree':
case 'refs':
$uri = $this->getPathURI("/{$action}/{$path}{$commit}{$line}");
break;
case 'compare':
$uri = $this->getPathURI("/{$action}/");
if (strlen($head)) {
$query['head'] = $head;
} else if (strlen($raw_commit)) {
$query['commit'] = $raw_commit;
} else if (strlen($raw_branch)) {
$query['head'] = $raw_branch;
}
if (strlen($against)) {
$query['against'] = $against;
}
break;
case 'branch':
if (strlen($path)) {
$uri = $this->getPathURI("/repository/{$path}");
} else {
$uri = $this->getPathURI('/');
}
break;
case 'external':
$commit = ltrim($commit, ';');
$uri = "/diffusion/external/{$commit}/";
break;
case 'rendering-ref':
// This isn't a real URI per se, it's passed as a query parameter to
// the ajax changeset stuff but then we parse it back out as though
// it came from a URI.
$uri = rawurldecode("{$path}{$commit}");
break;
case 'clone':
$uri = $this->getPathURI("/{$action}/");
break;
}
if ($action == 'rendering-ref') {
return $uri;
}
if (isset($params['lint'])) {
$params['params'] = idx($params, 'params', array()) + array(
'lint' => $params['lint'],
);
}
$query = idx($params, 'params', array()) + $query;
return new PhutilURI($uri, $query);
}
public function updateURIIndex() {
$indexes = array();
$uris = $this->getURIs();
foreach ($uris as $uri) {
if ($uri->getIsDisabled()) {
continue;
}
$indexes[] = $uri->getNormalizedURI();
}
PhabricatorRepositoryURIIndex::updateRepositoryURIs(
$this->getPHID(),
$indexes);
return $this;
}
public function isTracked() {
$status = $this->getDetail('tracking-enabled');
$map = self::getStatusMap();
$spec = idx($map, $status);
if (!$spec) {
if ($status) {
$status = self::STATUS_ACTIVE;
} else {
$status = self::STATUS_INACTIVE;
}
$spec = idx($map, $status);
}
return (bool)idx($spec, 'isTracked', false);
}
public function getDefaultBranch() {
$default = $this->getDetail('default-branch');
if (phutil_nonempty_string($default)) {
return $default;
}
$default_branches = array(
PhabricatorRepositoryType::REPOSITORY_TYPE_GIT => 'master',
PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL => 'default',
);
return idx($default_branches, $this->getVersionControlSystem());
}
public function getDefaultArcanistBranch() {
return coalesce($this->getDefaultBranch(), 'svn');
}
private function isBranchInFilter($branch, $filter_key) {
$vcs = $this->getVersionControlSystem();
$is_git = ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_GIT);
$use_filter = ($is_git);
if (!$use_filter) {
// If this VCS doesn't use filters, pass everything through.
return true;
}
$filter = $this->getDetail($filter_key, array());
// If there's no filter set, let everything through.
if (!$filter) {
return true;
}
// If this branch isn't literally named `regexp(...)`, and it's in the
// filter list, let it through.
if (isset($filter[$branch])) {
if (self::extractBranchRegexp($branch) === null) {
return true;
}
}
// If the branch matches a regexp, let it through.
foreach ($filter as $pattern => $ignored) {
$regexp = self::extractBranchRegexp($pattern);
if ($regexp !== null) {
if (preg_match($regexp, $branch)) {
return true;
}
}
}
// Nothing matched, so filter this branch out.
return false;
}
public static function extractBranchRegexp($pattern) {
$matches = null;
if (preg_match('/^regexp\\((.*)\\)\z/', $pattern, $matches)) {
return $matches[1];
}
return null;
}
public function shouldTrackRef(DiffusionRepositoryRef $ref) {
// At least for now, don't track the staging area tags.
if ($ref->isTag()) {
if (preg_match('(^phabricator/)', $ref->getShortName())) {
return false;
}
}
if (!$ref->isBranch()) {
return true;
}
return $this->shouldTrackBranch($ref->getShortName());
}
public function shouldTrackBranch($branch) {
return $this->isBranchInFilter($branch, 'branch-filter');
}
public function isBranchPermanentRef($branch) {
return $this->isBranchInFilter($branch, 'close-commits-filter');
}
public function formatCommitName($commit_identifier, $local = false) {
$vcs = $this->getVersionControlSystem();
$type_git = PhabricatorRepositoryType::REPOSITORY_TYPE_GIT;
$type_hg = PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL;
$is_git = ($vcs == $type_git);
$is_hg = ($vcs == $type_hg);
if ($is_git || $is_hg) {
$name = substr($commit_identifier, 0, 12);
$need_scope = false;
} else {
$name = $commit_identifier;
$need_scope = true;
}
if (!$local) {
$need_scope = true;
}
if ($need_scope) {
$callsign = $this->getCallsign();
if ($callsign) {
$scope = "r{$callsign}";
} else {
$id = $this->getID();
$scope = "R{$id}:";
}
$name = $scope.$name;
}
return $name;
}
public function isImporting() {
return (bool)$this->getDetail('importing', false);
}
public function isNewlyInitialized() {
return (bool)$this->getDetail('newly-initialized', false);
}
public function loadImportProgress() {
$progress = queryfx_all(
$this->establishConnection('r'),
'SELECT importStatus, count(*) N FROM %T WHERE repositoryID = %d
GROUP BY importStatus',
id(new PhabricatorRepositoryCommit())->getTableName(),
$this->getID());
$done = 0;
$total = 0;
foreach ($progress as $row) {
$total += $row['N'] * 3;
$status = $row['importStatus'];
if ($status & PhabricatorRepositoryCommit::IMPORTED_MESSAGE) {
$done += $row['N'];
}
if ($status & PhabricatorRepositoryCommit::IMPORTED_CHANGE) {
$done += $row['N'];
}
if ($status & PhabricatorRepositoryCommit::IMPORTED_PUBLISH) {
$done += $row['N'];
}
}
if ($total) {
$ratio = ($done / $total);
} else {
$ratio = 0;
}
// Cap this at "99.99%", because it's confusing to users when the actual
// fraction is "99.996%" and it rounds up to "100.00%".
if ($ratio > 0.9999) {
$ratio = 0.9999;
}
return $ratio;
}
/* -( Publishing )--------------------------------------------------------- */
public function newPublisher() {
return id(new PhabricatorRepositoryPublisher())
->setRepository($this);
}
public function isPublishingDisabled() {
return $this->getDetail('herald-disabled');
}
public function getPermanentRefRules() {
return array_keys($this->getDetail('close-commits-filter', array()));
}
public function setPermanentRefRules(array $rules) {
$rules = array_fill_keys($rules, true);
$this->setDetail('close-commits-filter', $rules);
return $this;
}
public function getTrackOnlyRules() {
return array_keys($this->getDetail('branch-filter', array()));
}
public function setTrackOnlyRules(array $rules) {
$rules = array_fill_keys($rules, true);
$this->setDetail('branch-filter', $rules);
return $this;
}
public function supportsFetchRules() {
if ($this->isGit()) {
return true;
}
return false;
}
public function getFetchRules() {
return $this->getDetail('fetch-rules', array());
}
public function setFetchRules(array $rules) {
return $this->setDetail('fetch-rules', $rules);
}
/* -( Repository URI Management )------------------------------------------ */
/**
* Get the remote URI for this repository.
*
* @return string
* @task uri
*/
public function getRemoteURI() {
return (string)$this->getRemoteURIObject();
}
/**
* Get the remote URI for this repository, including credentials if they're
* used by this repository.
*
* @return PhutilOpaqueEnvelope URI, possibly including credentials.
* @task uri
*/
public function getRemoteURIEnvelope() {
$uri = $this->getRemoteURIObject();
$remote_protocol = $this->getRemoteProtocol();
if ($remote_protocol == 'http' || $remote_protocol == 'https') {
// For SVN, we use `--username` and `--password` flags separately, so
// don't add any credentials here.
if (!$this->isSVN()) {
$credential_phid = $this->getCredentialPHID();
if ($credential_phid) {
$key = PassphrasePasswordKey::loadFromPHID(
$credential_phid,
PhabricatorUser::getOmnipotentUser());
$uri->setUser($key->getUsernameEnvelope()->openEnvelope());
$uri->setPass($key->getPasswordEnvelope()->openEnvelope());
}
}
}
return new PhutilOpaqueEnvelope((string)$uri);
}
/**
* Get the clone (or checkout) URI for this repository, without authentication
* information.
*
* @return string Repository URI.
* @task uri
*/
public function getPublicCloneURI() {
return (string)$this->getCloneURIObject();
}
/**
* Get the protocol for the repository's remote.
*
* @return string Protocol, like "ssh" or "git".
* @task uri
*/
public function getRemoteProtocol() {
$uri = $this->getRemoteURIObject();
return $uri->getProtocol();
}
/**
* Get a parsed object representation of the repository's remote URI..
*
* @return wild A @{class@arcanist:PhutilURI}.
* @task uri
*/
public function getRemoteURIObject() {
$raw_uri = $this->getDetail('remote-uri');
if (!strlen($raw_uri)) {
return new PhutilURI('');
}
if (!strncmp($raw_uri, '/', 1)) {
return new PhutilURI('file://'.$raw_uri);
}
return new PhutilURI($raw_uri);
}
/**
* Get the "best" clone/checkout URI for this repository, on any protocol.
*/
public function getCloneURIObject() {
if (!$this->isHosted()) {
if ($this->isSVN()) {
// Make sure we pick up the "Import Only" path for Subversion, so
// the user clones the repository starting at the correct path, not
// from the root.
$base_uri = $this->getSubversionBaseURI();
$base_uri = new PhutilURI($base_uri);
$path = $base_uri->getPath();
if (!$path) {
$path = '/';
}
// If the trailing "@" is not required to escape the URI, strip it for
// readability.
if (!preg_match('/@.*@/', $path)) {
$path = rtrim($path, '@');
}
$base_uri->setPath($path);
return $base_uri;
} else {
return $this->getRemoteURIObject();
}
}
// TODO: This should be cleaned up to deal with all the new URI handling.
$another_copy = id(new PhabricatorRepositoryQuery())
->setViewer(PhabricatorUser::getOmnipotentUser())
->withPHIDs(array($this->getPHID()))
->needURIs(true)
->executeOne();
$clone_uris = $another_copy->getCloneURIs();
if (!$clone_uris) {
return null;
}
return head($clone_uris)->getEffectiveURI();
}
private function getRawHTTPCloneURIObject() {
$uri = PhabricatorEnv::getProductionURI($this->getURI());
$uri = new PhutilURI($uri);
if ($this->isGit()) {
$uri->setPath($uri->getPath().$this->getCloneName().'.git');
} else if ($this->isHg()) {
$uri->setPath($uri->getPath().$this->getCloneName().'/');
}
return $uri;
}
/**
* Determine if we should connect to the remote using SSH flags and
* credentials.
*
* @return bool True to use the SSH protocol.
* @task uri
*/
private function shouldUseSSH() {
if ($this->isHosted()) {
return false;
}
$protocol = $this->getRemoteProtocol();
if ($this->isSSHProtocol($protocol)) {
return true;
}
return false;
}
/**
* Determine if we should connect to the remote using HTTP flags and
* credentials.
*
* @return bool True to use the HTTP protocol.
* @task uri
*/
private function shouldUseHTTP() {
if ($this->isHosted()) {
return false;
}
$protocol = $this->getRemoteProtocol();
return ($protocol == 'http' || $protocol == 'https');
}
/**
* Determine if we should connect to the remote using SVN flags and
* credentials.
*
* @return bool True to use the SVN protocol.
* @task uri
*/
private function shouldUseSVNProtocol() {
if ($this->isHosted()) {
return false;
}
$protocol = $this->getRemoteProtocol();
return ($protocol == 'svn');
}
/**
* Determine if a protocol is SSH or SSH-like.
*
* @param string A protocol string, like "http" or "ssh".
* @return bool True if the protocol is SSH-like.
* @task uri
*/
private function isSSHProtocol($protocol) {
return ($protocol == 'ssh' || $protocol == 'svn+ssh');
}
public function delete() {
$this->openTransaction();
$paths = id(new PhabricatorOwnersPath())
->loadAllWhere('repositoryPHID = %s', $this->getPHID());
foreach ($paths as $path) {
$path->delete();
}
queryfx(
$this->establishConnection('w'),
'DELETE FROM %T WHERE repositoryPHID = %s',
id(new PhabricatorRepositorySymbol())->getTableName(),
$this->getPHID());
$commits = id(new PhabricatorRepositoryCommit())
->loadAllWhere('repositoryID = %d', $this->getID());
foreach ($commits as $commit) {
// note PhabricatorRepositoryAuditRequests and
// PhabricatorRepositoryCommitData are deleted here too.
$commit->delete();
}
$uris = id(new PhabricatorRepositoryURI())
->loadAllWhere('repositoryPHID = %s', $this->getPHID());
foreach ($uris as $uri) {
$uri->delete();
}
$ref_cursors = id(new PhabricatorRepositoryRefCursor())
->loadAllWhere('repositoryPHID = %s', $this->getPHID());
foreach ($ref_cursors as $cursor) {
$cursor->delete();
}
$conn_w = $this->establishConnection('w');
queryfx(
$conn_w,
'DELETE FROM %T WHERE repositoryID = %d',
self::TABLE_FILESYSTEM,
$this->getID());
queryfx(
$conn_w,
'DELETE FROM %T WHERE repositoryID = %d',
self::TABLE_PATHCHANGE,
$this->getID());
queryfx(
$conn_w,
'DELETE FROM %T WHERE repositoryID = %d',
self::TABLE_SUMMARY,
$this->getID());
$result = parent::delete();
$this->saveTransaction();
return $result;
}
public function isGit() {
$vcs = $this->getVersionControlSystem();
return ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_GIT);
}
public function isSVN() {
$vcs = $this->getVersionControlSystem();
return ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_SVN);
}
public function isHg() {
$vcs = $this->getVersionControlSystem();
return ($vcs == PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL);
}
public function isHosted() {
return (bool)$this->getDetail('hosting-enabled', false);
}
public function setHosted($enabled) {
return $this->setDetail('hosting-enabled', $enabled);
}
public function canServeProtocol(
$protocol,
$write,
$is_intracluster = false) {
// See T13192. If a repository is inactive, don't serve it to users. We
// still synchronize it within the cluster and serve it to other repository
// nodes.
if (!$is_intracluster) {
if (!$this->isTracked()) {
return false;
}
}
$clone_uris = $this->getCloneURIs();
foreach ($clone_uris as $uri) {
if ($uri->getBuiltinProtocol() !== $protocol) {
continue;
}
$io_type = $uri->getEffectiveIoType();
if ($io_type == PhabricatorRepositoryURI::IO_READWRITE) {
return true;
}
if (!$write) {
if ($io_type == PhabricatorRepositoryURI::IO_READ) {
return true;
}
}
}
if ($write) {
if ($this->isReadOnly()) {
return false;
}
}
return false;
}
public function hasLocalWorkingCopy() {
try {
self::assertLocalExists();
return true;
} catch (Exception $ex) {
return false;
}
}
/**
* Raise more useful errors when there are basic filesystem problems.
*/
private function assertLocalExists() {
if (!$this->usesLocalWorkingCopy()) {
return;
}
$local = $this->getLocalPath();
Filesystem::assertExists($local);
Filesystem::assertIsDirectory($local);
Filesystem::assertReadable($local);
}
/**
* Determine if the working copy is bare or not. In Git, this corresponds
* to `--bare`. In Mercurial, `--noupdate`.
*/
public function isWorkingCopyBare() {
switch ($this->getVersionControlSystem()) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
return false;
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
$local = $this->getLocalPath();
if (Filesystem::pathExists($local.'/.git')) {
return false;
} else {
return true;
}
}
}
public function usesLocalWorkingCopy() {
switch ($this->getVersionControlSystem()) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
return $this->isHosted();
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
return true;
}
}
public function getHookDirectories() {
$directories = array();
if (!$this->isHosted()) {
return $directories;
}
$root = $this->getLocalPath();
switch ($this->getVersionControlSystem()) {
case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
if ($this->isWorkingCopyBare()) {
$directories[] = $root.'/hooks/pre-receive-phabricator.d/';
} else {
$directories[] = $root.'/.git/hooks/pre-receive-phabricator.d/';
}
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
$directories[] = $root.'/hooks/pre-commit-phabricator.d/';
break;
case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
// NOTE: We don't support custom Mercurial hooks for now because they're
// messy and we can't easily just drop a `hooks.d/` directory next to
// the hooks.
break;
}
return $directories;
}
public function canDestroyWorkingCopy() {
if ($this->isHosted()) {
// Never destroy hosted working copies.
return false;
}
$default_path = PhabricatorEnv::getEnvConfig(
'repository.default-local-path');
return Filesystem::isDescendant($this->getLocalPath(), $default_path);
}
public function canUsePathTree() {
return !$this->isSVN();
}
public function canUseGitLFS() {
if (!$this->isGit()) {
return false;
}
if (!$this->isHosted()) {
return false;
}
if (!PhabricatorEnv::getEnvConfig('diffusion.allow-git-lfs')) {
return false;
}
return true;
}
public function getGitLFSURI($path = null) {
if (!$this->canUseGitLFS()) {
throw new Exception(
pht(
'This repository does not support Git LFS, so Git LFS URIs can '.
'not be generated for it.'));
}
$uri = $this->getRawHTTPCloneURIObject();
$uri = (string)$uri;
$uri = $uri.'/'.$path;
return $uri;
}
public function canMirror() {
if ($this->isGit() || $this->isHg()) {
return true;
}
return false;
}
public function canAllowDangerousChanges() {
if (!$this->isHosted()) {
return false;
}
// In Git and Mercurial, ref deletions and rewrites are dangerous.
// In Subversion, editing revprops is dangerous.
return true;
}
public function shouldAllowDangerousChanges() {
return (bool)$this->getDetail('allow-dangerous-changes');
}
public function canAllowEnormousChanges() {
if (!$this->isHosted()) {
return false;
}
return true;
}
public function shouldAllowEnormousChanges() {
return (bool)$this->getDetail('allow-enormous-changes');
}
public function writeStatusMessage(
$status_type,
$status_code,
array $parameters = array()) {
$table = new PhabricatorRepositoryStatusMessage();
$conn_w = $table->establishConnection('w');
$table_name = $table->getTableName();
if ($status_code === null) {
queryfx(
$conn_w,
'DELETE FROM %T WHERE repositoryID = %d AND statusType = %s',
$table_name,
$this->getID(),
$status_type);
} else {
// If the existing message has the same code (e.g., we just hit an
// error and also previously hit an error) we increment the message
// count. This allows us to determine how many times in a row we've
// run into an error.
// NOTE: The assignments in "ON DUPLICATE KEY UPDATE" are evaluated
// in order, so the "messageCount" assignment must occur before the
// "statusCode" assignment. See T11705.
queryfx(
$conn_w,
'INSERT INTO %T
(repositoryID, statusType, statusCode, parameters, epoch,
messageCount)
VALUES (%d, %s, %s, %s, %d, %d)
ON DUPLICATE KEY UPDATE
messageCount =
IF(
statusCode = VALUES(statusCode),
messageCount + VALUES(messageCount),
VALUES(messageCount)),
statusCode = VALUES(statusCode),
parameters = VALUES(parameters),
epoch = VALUES(epoch)',
$table_name,
$this->getID(),
$status_type,
$status_code,
json_encode($parameters),
time(),
1);
}
return $this;
}
public static function assertValidRemoteURI($uri) {
if (trim($uri) != $uri) {
throw new Exception(
pht('The remote URI has leading or trailing whitespace.'));
}
$uri_object = new PhutilURI($uri);
$protocol = $uri_object->getProtocol();
// Catch confusion between Git/SCP-style URIs and normal URIs. See T3619
// for discussion. This is usually a user adding "ssh://" to an implicit
// SSH Git URI.
if ($protocol == 'ssh') {
if (preg_match('(^[^:@]+://[^/:]+:[^\d])', $uri)) {
throw new Exception(
pht(
"The remote URI is not formatted correctly. Remote URIs ".
"with an explicit protocol should be in the form ".
"'%s', not '%s'. The '%s' syntax is only valid in SCP-style URIs.",
'proto://domain/path',
'proto://domain:/path',
':/path'));
}
}
switch ($protocol) {
case 'ssh':
case 'http':
case 'https':
case 'git':
case 'svn':
case 'svn+ssh':
break;
default:
// NOTE: We're explicitly rejecting 'file://' because it can be
// used to clone from the working copy of another repository on disk
// that you don't normally have permission to access.
throw new Exception(
pht(
'The URI protocol is unrecognized. It should begin with '.
'"%s", "%s", "%s", "%s", "%s", "%s", or be in the form "%s".',
'ssh://',
'http://',
'https://',
'git://',
'svn://',
'svn+ssh://',
'git@domain.com:path'));
}
return true;
}
/**
* Load the pull frequency for this repository, based on the time since the
* last activity.
*
* We pull rarely used repositories less frequently. This finds the most
* recent commit which is older than the current time (which prevents us from
* spinning on repositories with a silly commit post-dated to some time in
* 2037). We adjust the pull frequency based on when the most recent commit
* occurred.
*
* @param int The minimum update interval to use, in seconds.
* @return int Repository update interval, in seconds.
*/
public function loadUpdateInterval($minimum = 15) {
// First, check if we've hit errors recently. If we have, wait one period
// for each consecutive error. Normally, this corresponds to a backoff of
// 15s, 30s, 45s, etc.
$message_table = new PhabricatorRepositoryStatusMessage();
$conn = $message_table->establishConnection('r');
$error_count = queryfx_one(
$conn,
'SELECT MAX(messageCount) error_count FROM %T
WHERE repositoryID = %d
AND statusType IN (%Ls)
AND statusCode IN (%Ls)',
$message_table->getTableName(),
$this->getID(),
array(
PhabricatorRepositoryStatusMessage::TYPE_INIT,
PhabricatorRepositoryStatusMessage::TYPE_FETCH,
),
array(
PhabricatorRepositoryStatusMessage::CODE_ERROR,
));
$error_count = (int)$error_count['error_count'];
if ($error_count > 0) {
return (int)($minimum * $error_count);
}
// If a repository is still importing, always pull it as frequently as
// possible. This prevents us from hanging for a long time at 99.9% when
// importing an inactive repository.
if ($this->isImporting()) {
return $minimum;
}
$window_start = (PhabricatorTime::getNow() + $minimum);
$table = id(new PhabricatorRepositoryCommit());
$last_commit = queryfx_one(
$table->establishConnection('r'),
'SELECT epoch FROM %T
WHERE repositoryID = %d AND epoch <= %d
ORDER BY epoch DESC LIMIT 1',
$table->getTableName(),
$this->getID(),
$window_start);
if ($last_commit) {
$time_since_commit = ($window_start - $last_commit['epoch']);
} else {
// If the repository has no commits, treat the creation date as
// though it were the date of the last commit. This makes empty
// repositories update quickly at first but slow down over time
// if they don't see any activity.
$time_since_commit = ($window_start - $this->getDateCreated());
}
$last_few_days = phutil_units('3 days in seconds');
if ($time_since_commit <= $last_few_days) {
// For repositories with activity in the recent past, we wait one
// extra second for every 10 minutes since the last commit. This
// shorter backoff is intended to handle weekends and other short
// breaks from development.
$smart_wait = ($time_since_commit / 600);
} else {
// For repositories without recent activity, we wait one extra second
// for every 4 minutes since the last commit. This longer backoff
// handles rarely used repositories, up to the maximum.
$smart_wait = ($time_since_commit / 240);
}
// We'll never wait more than 6 hours to pull a repository.
$longest_wait = phutil_units('6 hours in seconds');
$smart_wait = min($smart_wait, $longest_wait);
$smart_wait = max($minimum, $smart_wait);
return (int)$smart_wait;
}
/**
* Time limit for cloning or copying this repository.
*
* This limit is used to timeout operations like `git clone` or `git fetch`
* when doing intracluster synchronization, building working copies, etc.
*
* @return int Maximum number of seconds to spend copying this repository.
*/
public function getCopyTimeLimit() {
return $this->getDetail('limit.copy');
}
public function setCopyTimeLimit($limit) {
return $this->setDetail('limit.copy', $limit);
}
public function getDefaultCopyTimeLimit() {
return phutil_units('15 minutes in seconds');
}
public function getEffectiveCopyTimeLimit() {
$limit = $this->getCopyTimeLimit();
if ($limit) {
return $limit;
}
return $this->getDefaultCopyTimeLimit();
}
public function getFilesizeLimit() {
return $this->getDetail('limit.filesize');
}
public function setFilesizeLimit($limit) {
return $this->setDetail('limit.filesize', $limit);
}
public function getTouchLimit() {
return $this->getDetail('limit.touch');
}
public function setTouchLimit($limit) {
return $this->setDetail('limit.touch', $limit);
}
/**
* Retrieve the service URI for the device hosting this repository.
*
* See @{method:newConduitClient} for a general discussion of interacting
* with repository services. This method provides lower-level resolution of
* services, returning raw URIs.
*
* @param PhabricatorUser Viewing user.
* @param map<string, wild> Constraints on selectable services.
* @return string|null URI, or `null` for local repositories.
*/
public function getAlmanacServiceURI(
PhabricatorUser $viewer,
array $options) {
$refs = $this->getAlmanacServiceRefs($viewer, $options);
if (!$refs) {
return null;
}
$ref = head($refs);
return $ref->getURI();
}
public function getAlmanacServiceRefs(
PhabricatorUser $viewer,
array $options) {
PhutilTypeSpec::checkMap(
$options,
array(
'neverProxy' => 'bool',
'protocols' => 'list<string>',
'writable' => 'optional bool',
));
$never_proxy = $options['neverProxy'];
$protocols = $options['protocols'];
$writable = idx($options, 'writable', false);
$cache_key = $this->getAlmanacServiceCacheKey();
if (!$cache_key) {
return array();
}
$cache = PhabricatorCaches::getMutableStructureCache();
$uris = $cache->getKey($cache_key, false);
// If we haven't built the cache yet, build it now.
if ($uris === false) {
$uris = $this->buildAlmanacServiceURIs();
$cache->setKey($cache_key, $uris);
}
if ($uris === null) {
return array();
}
$local_device = AlmanacKeys::getDeviceID();
if ($never_proxy && !$local_device) {
throw new Exception(
pht(
'Unable to handle proxied service request. This device is not '.
'registered, so it can not identify local services. Register '.
'this device before sending requests here.'));
}
$protocol_map = array_fuse($protocols);
$results = array();
foreach ($uris as $uri) {
// If we're never proxying this and it's locally satisfiable, return
// `null` to tell the caller to handle it locally. If we're allowed to
// proxy, we skip this check and may proxy the request to ourselves.
// (That proxied request will end up here with proxying forbidden,
// return `null`, and then the request will actually run.)
if ($local_device && $never_proxy) {
if ($uri['device'] == $local_device) {
return array();
}
}
if (isset($protocol_map[$uri['protocol']])) {
$results[] = $uri;
}
}
if (!$results) {
throw new Exception(
pht(
'The Almanac service for this repository is not bound to any '.
'interfaces which support the required protocols (%s).',
implode(', ', $protocols)));
}
if ($never_proxy) {
// See PHI1030. This error can arise from various device name/address
// mismatches which are hard to detect, so try to provide as much
// information as we can.
if ($writable) {
$request_type = pht('(This is a write request.)');
} else {
$request_type = pht('(This is a read request.)');
}
throw new Exception(
pht(
'This repository request (for repository "%s") has been '.
'incorrectly routed to a cluster host (with device name "%s", '.
'and hostname "%s") which can not serve the request.'.
"\n\n".
'The Almanac device address for the correct device may improperly '.
'point at this host, or the "device.id" configuration file on '.
'this host may be incorrect.'.
"\n\n".
'Requests routed within the cluster are always '.
'expected to be sent to a node which can serve the request. To '.
'prevent loops, this request will not be proxied again.'.
"\n\n".
"%s",
$this->getDisplayName(),
$local_device,
php_uname('n'),
$request_type));
}
if (count($results) > 1) {
if (!$this->supportsSynchronization()) {
throw new Exception(
pht(
'Repository "%s" is bound to multiple active repository hosts, '.
'but this repository does not support cluster synchronization. '.
'Declusterize this repository or move it to a service with only '.
'one host.',
$this->getDisplayName()));
}
}
$refs = array();
foreach ($results as $result) {
$refs[] = DiffusionServiceRef::newFromDictionary($result);
}
// If we require a writable device, remove URIs which aren't writable.
if ($writable) {
foreach ($refs as $key => $ref) {
if (!$ref->isWritable()) {
unset($refs[$key]);
}
}
if (!$refs) {
throw new Exception(
pht(
'This repository ("%s") is not writable with the given '.
'protocols (%s). The Almanac service for this repository has no '.
'writable bindings that support these protocols.',
$this->getDisplayName(),
implode(', ', $protocols)));
}
}
if ($writable) {
$refs = $this->sortWritableAlmanacServiceRefs($refs);
} else {
$refs = $this->sortReadableAlmanacServiceRefs($refs);
}
return array_values($refs);
}
private function sortReadableAlmanacServiceRefs(array $refs) {
assert_instances_of($refs, 'DiffusionServiceRef');
shuffle($refs);
return $refs;
}
private function sortWritableAlmanacServiceRefs(array $refs) {
assert_instances_of($refs, 'DiffusionServiceRef');
// See T13109 for discussion of how this method routes requests.
// In the absence of other rules, we'll send traffic to devices randomly.
// We also want to select randomly among nodes which are equally good
// candidates to receive the write, and accomplish that by shuffling the
// list up front.
shuffle($refs);
$order = array();
// If some device is currently holding the write lock, send all requests
// to that device. We're trying to queue writes on a single device so they
// do not need to wait for read synchronization after earlier writes
// complete.
$writer = PhabricatorRepositoryWorkingCopyVersion::loadWriter(
$this->getPHID());
if ($writer) {
$device_phid = $writer->getWriteProperty('devicePHID');
foreach ($refs as $key => $ref) {
if ($ref->getDevicePHID() === $device_phid) {
$order[] = $key;
}
}
}
// If no device is currently holding the write lock, try to send requests
// to a device which is already up to date and will not need to synchronize
// before it can accept the write.
$versions = PhabricatorRepositoryWorkingCopyVersion::loadVersions(
$this->getPHID());
if ($versions) {
$max_version = (int)max(mpull($versions, 'getRepositoryVersion'));
$max_devices = array();
foreach ($versions as $version) {
if ($version->getRepositoryVersion() == $max_version) {
$max_devices[] = $version->getDevicePHID();
}
}
$max_devices = array_fuse($max_devices);
foreach ($refs as $key => $ref) {
if (isset($max_devices[$ref->getDevicePHID()])) {
$order[] = $key;
}
}
}
// Reorder the results, putting any we've selected as preferred targets for
// the write at the head of the list.
$refs = array_select_keys($refs, $order) + $refs;
return $refs;
}
public function supportsSynchronization() {
// TODO: For now, this is only supported for Git.
if (!$this->isGit()) {
return false;
}
return true;
}
public function supportsRefs() {
if ($this->isSVN()) {
return false;
}
return true;
}
public function getAlmanacServiceCacheKey() {
$service_phid = $this->getAlmanacServicePHID();
if (!$service_phid) {
return null;
}
$repository_phid = $this->getPHID();
$parts = array(
"repo({$repository_phid})",
"serv({$service_phid})",
'v4',
);
return implode('.', $parts);
}
private function buildAlmanacServiceURIs() {
$service = $this->loadAlmanacService();
if (!$service) {
return null;
}
$bindings = $service->getActiveBindings();
if (!$bindings) {
throw new Exception(
pht(
'The Almanac service for this repository is not bound to any '.
'active interfaces.'));
}
$uris = array();
foreach ($bindings as $binding) {
$iface = $binding->getInterface();
$uri = $this->getClusterRepositoryURIFromBinding($binding);
$protocol = $uri->getProtocol();
$device_name = $iface->getDevice()->getName();
$device_phid = $iface->getDevice()->getPHID();
$uris[] = array(
'protocol' => $protocol,
'uri' => (string)$uri,
'device' => $device_name,
'writable' => (bool)$binding->getAlmanacPropertyValue('writable'),
'devicePHID' => $device_phid,
);
}
return $uris;
}
/**
* Build a new Conduit client in order to make a service call to this
* repository.
*
* If the repository is hosted locally, this method may return `null`. The
* caller should use `ConduitCall` or other local logic to complete the
* request.
*
* By default, we will return a @{class:ConduitClient} for any repository with
* a service, even if that service is on the current device.
*
* We do this because this configuration does not make very much sense in a
* production context, but is very common in a test/development context
* (where the developer's machine is both the web host and the repository
* service). By proxying in development, we get more consistent behavior
* between development and production, and don't have a major untested
* codepath.
*
* The `$never_proxy` parameter can be used to prevent this local proxying.
* If the flag is passed:
*
* - The method will return `null` (implying a local service call)
* if the repository service is hosted on the current device.
* - The method will throw if it would need to return a client.
*
* This is used to prevent loops in Conduit: the first request will proxy,
* even in development, but the second request will be identified as a
* cluster request and forced not to proxy.
*
* For lower-level service resolution, see @{method:getAlmanacServiceURI}.
*
* @param PhabricatorUser Viewing user.
* @param bool `true` to throw if a client would be returned.
* @return ConduitClient|null Client, or `null` for local repositories.
*/
public function newConduitClient(
PhabricatorUser $viewer,
$never_proxy = false) {
$uri = $this->getAlmanacServiceURI(
$viewer,
array(
'neverProxy' => $never_proxy,
'protocols' => array(
'http',
'https',
),
// At least today, no Conduit call can ever write to a repository,
// so it's fine to send anything to a read-only node.
'writable' => false,
));
if ($uri === null) {
return null;
}
$domain = id(new PhutilURI(PhabricatorEnv::getURI('/')))->getDomain();
$client = id(new ConduitClient($uri))
->setHost($domain);
if ($viewer->isOmnipotent()) {
// If the caller is the omnipotent user (normally, a daemon), we will
// sign the request with this host's asymmetric keypair.
$public_path = AlmanacKeys::getKeyPath('device.pub');
try {
$public_key = Filesystem::readFile($public_path);
} catch (Exception $ex) {
throw new PhutilAggregateException(
pht(
'Unable to read device public key while attempting to make '.
'authenticated method call within the cluster. '.
'Use `%s` to register keys for this device. Exception: %s',
'bin/almanac register',
$ex->getMessage()),
array($ex));
}
$private_path = AlmanacKeys::getKeyPath('device.key');
try {
$private_key = Filesystem::readFile($private_path);
$private_key = new PhutilOpaqueEnvelope($private_key);
} catch (Exception $ex) {
throw new PhutilAggregateException(
pht(
'Unable to read device private key while attempting to make '.
'authenticated method call within the cluster. '.
'Use `%s` to register keys for this device. Exception: %s',
'bin/almanac register',
$ex->getMessage()),
array($ex));
}
$client->setSigningKeys($public_key, $private_key);
} else {
// If the caller is a normal user, we generate or retrieve a cluster
// API token.
$token = PhabricatorConduitToken::loadClusterTokenForUser($viewer);
if ($token) {
$client->setConduitToken($token->getToken());
}
}
return $client;
}
public function newConduitClientForRequest(ConduitAPIRequest $request) {
// Figure out whether we're going to handle this request on this device,
// or proxy it to another node in the cluster.
// If this is a cluster request and we need to proxy, we'll explode here
// to prevent infinite recursion.
$viewer = $request->getViewer();
$is_cluster_request = $request->getIsClusterRequest();
$client = $this->newConduitClient(
$viewer,
$is_cluster_request);
return $client;
}
public function newConduitFuture(
PhabricatorUser $viewer,
$method,
array $params,
$never_proxy = false) {
$client = $this->newConduitClient(
$viewer,
$never_proxy);
if (!$client) {
$conduit_call = id(new ConduitCall($method, $params))
->setUser($viewer);
$future = new MethodCallFuture($conduit_call, 'execute');
} else {
$future = $client->callMethod($method, $params);
}
return $future;
}
public function getPassthroughEnvironmentalVariables() {
$env = $_ENV;
if ($this->isGit()) {
// $_ENV does not populate in CLI contexts if "E" is missing from
// "variables_order" in PHP config. Currently, we do not require this
// to be configured. Since it may not be, explicitly bring expected Git
// environmental variables into scope. This list is not exhaustive, but
// only lists variables with a known impact on commit hook behavior.
// This can be removed if we later require "E" in "variables_order".
$git_env = array(
'GIT_OBJECT_DIRECTORY',
'GIT_ALTERNATE_OBJECT_DIRECTORIES',
'GIT_QUARANTINE_PATH',
);
foreach ($git_env as $key) {
$value = getenv($key);
if (strlen($value)) {
$env[$key] = $value;
}
}
$key = 'GIT_PUSH_OPTION_COUNT';
$git_count = getenv($key);
if (strlen($git_count)) {
$git_count = (int)$git_count;
$env[$key] = $git_count;
for ($ii = 0; $ii < $git_count; $ii++) {
$key = 'GIT_PUSH_OPTION_'.$ii;
$env[$key] = getenv($key);
}
}
}
$result = array();
foreach ($env as $key => $value) {
// In Git, pass anything matching "GIT_*" though. Some of these variables
// need to be preserved to allow `git` operations to work properly when
// running from commit hooks.
if ($this->isGit()) {
if (preg_match('/^GIT_/', $key)) {
$result[$key] = $value;
}
}
}
return $result;
}
public function supportsBranchComparison() {
return $this->isGit();
}
public function isReadOnly() {
return (bool)$this->getDetail('read-only');
}
public function setReadOnly($read_only) {
return $this->setDetail('read-only', $read_only);
}
public function getReadOnlyMessage() {
return $this->getDetail('read-only-message');
}
public function setReadOnlyMessage($message) {
return $this->setDetail('read-only-message', $message);
}
public function getReadOnlyMessageForDisplay() {
$parts = array();
$parts[] = pht(
'This repository is currently in read-only maintenance mode.');
$message = $this->getReadOnlyMessage();
if ($message !== null) {
$parts[] = $message;
}
return implode("\n\n", $parts);
}
/* -( Repository URIs )---------------------------------------------------- */
public function attachURIs(array $uris) {
$custom_map = array();
foreach ($uris as $key => $uri) {
$builtin_key = $uri->getRepositoryURIBuiltinKey();
if ($builtin_key !== null) {
$custom_map[$builtin_key] = $key;
}
}
$builtin_uris = $this->newBuiltinURIs();
$seen_builtins = array();
foreach ($builtin_uris as $builtin_uri) {
$builtin_key = $builtin_uri->getRepositoryURIBuiltinKey();
$seen_builtins[$builtin_key] = true;
// If this builtin URI is disabled, don't attach it and remove the
// persisted version if it exists.
if ($builtin_uri->getIsDisabled()) {
if (isset($custom_map[$builtin_key])) {
unset($uris[$custom_map[$builtin_key]]);
}
continue;
}
// If the URI exists, make sure it's marked as not being disabled.
if (isset($custom_map[$builtin_key])) {
$uris[$custom_map[$builtin_key]]->setIsDisabled(false);
}
}
// Remove any builtins which no longer exist.
foreach ($custom_map as $builtin_key => $key) {
if (empty($seen_builtins[$builtin_key])) {
unset($uris[$key]);
}
}
$this->uris = $uris;
return $this;
}
public function getURIs() {
return $this->assertAttached($this->uris);
}
public function getCloneURIs() {
$uris = $this->getURIs();
$clone = array();
foreach ($uris as $uri) {
if (!$uri->isBuiltin()) {
continue;
}
if ($uri->getIsDisabled()) {
continue;
}
$io_type = $uri->getEffectiveIoType();
$is_clone =
($io_type == PhabricatorRepositoryURI::IO_READ) ||
($io_type == PhabricatorRepositoryURI::IO_READWRITE);
if (!$is_clone) {
continue;
}
$clone[] = $uri;
}
$clone = msort($clone, 'getURIScore');
$clone = array_reverse($clone);
return $clone;
}
public function newBuiltinURIs() {
$has_callsign = ($this->getCallsign() !== null);
$has_shortname = ($this->getRepositorySlug() !== null);
$identifier_map = array(
PhabricatorRepositoryURI::BUILTIN_IDENTIFIER_CALLSIGN => $has_callsign,
PhabricatorRepositoryURI::BUILTIN_IDENTIFIER_SHORTNAME => $has_shortname,
PhabricatorRepositoryURI::BUILTIN_IDENTIFIER_ID => true,
);
// If the view policy of the repository is public, support anonymous HTTP
// even if authenticated HTTP is not supported.
if ($this->getViewPolicy() === PhabricatorPolicies::POLICY_PUBLIC) {
$allow_http = true;
} else {
$allow_http = PhabricatorEnv::getEnvConfig('diffusion.allow-http-auth');
}
$base_uri = PhabricatorEnv::getURI('/');
$base_uri = new PhutilURI($base_uri);
$has_https = ($base_uri->getProtocol() == 'https');
$has_https = ($has_https && $allow_http);
$has_http = !PhabricatorEnv::getEnvConfig('security.require-https');
$has_http = ($has_http && $allow_http);
// HTTP is not supported for Subversion.
if ($this->isSVN()) {
$has_http = false;
$has_https = false;
}
- $has_ssh = (bool)strlen(PhabricatorEnv::getEnvConfig('phd.user'));
+ $phd_user = PhabricatorEnv::getEnvConfig('phd.user');
+ $has_ssh = phutil_nonempty_string($phd_user);
$protocol_map = array(
PhabricatorRepositoryURI::BUILTIN_PROTOCOL_SSH => $has_ssh,
PhabricatorRepositoryURI::BUILTIN_PROTOCOL_HTTPS => $has_https,
PhabricatorRepositoryURI::BUILTIN_PROTOCOL_HTTP => $has_http,
);
$uris = array();
foreach ($protocol_map as $protocol => $proto_supported) {
foreach ($identifier_map as $identifier => $id_supported) {
// This is just a dummy value because it can't be empty; we'll force
// it to a proper value when using it in the UI.
$builtin_uri = "{$protocol}://{$identifier}";
$uris[] = PhabricatorRepositoryURI::initializeNewURI()
->setRepositoryPHID($this->getPHID())
->attachRepository($this)
->setBuiltinProtocol($protocol)
->setBuiltinIdentifier($identifier)
->setURI($builtin_uri)
->setIsDisabled((int)(!$proto_supported || !$id_supported));
}
}
return $uris;
}
public function getClusterRepositoryURIFromBinding(
AlmanacBinding $binding) {
$protocol = $binding->getAlmanacPropertyValue('protocol');
if ($protocol === null) {
$protocol = 'https';
}
$iface = $binding->getInterface();
$address = $iface->renderDisplayAddress();
$path = $this->getURI();
return id(new PhutilURI("{$protocol}://{$address}"))
->setPath($path);
}
public function loadAlmanacService() {
$service_phid = $this->getAlmanacServicePHID();
if (!$service_phid) {
// No service, so this is a local repository.
return null;
}
$service = id(new AlmanacServiceQuery())
->setViewer(PhabricatorUser::getOmnipotentUser())
->withPHIDs(array($service_phid))
->needActiveBindings(true)
->needProperties(true)
->executeOne();
if (!$service) {
throw new Exception(
pht(
'The Almanac service for this repository is invalid or could not '.
'be loaded.'));
}
$service_type = $service->getServiceImplementation();
if (!($service_type instanceof AlmanacClusterRepositoryServiceType)) {
throw new Exception(
pht(
'The Almanac service for this repository does not have the correct '.
'service type.'));
}
return $service;
}
public function markImporting() {
$this->openTransaction();
$this->beginReadLocking();
$repository = $this->reload();
$repository->setDetail('importing', true);
$repository->save();
$this->endReadLocking();
$this->saveTransaction();
return $repository;
}
/* -( Symbols )-------------------------------------------------------------*/
public function getSymbolSources() {
return $this->getDetail('symbol-sources', array());
}
public function getSymbolLanguages() {
return $this->getDetail('symbol-languages', array());
}
/* -( Staging )------------------------------------------------------------ */
public function supportsStaging() {
return $this->isGit();
}
public function getStagingURI() {
if (!$this->supportsStaging()) {
return null;
}
return $this->getDetail('staging-uri', null);
}
/* -( Automation )--------------------------------------------------------- */
public function supportsAutomation() {
return $this->isGit();
}
public function canPerformAutomation() {
if (!$this->supportsAutomation()) {
return false;
}
if (!$this->getAutomationBlueprintPHIDs()) {
return false;
}
return true;
}
public function getAutomationBlueprintPHIDs() {
if (!$this->supportsAutomation()) {
return array();
}
return $this->getDetail('automation.blueprintPHIDs', array());
}
/* -( PhabricatorApplicationTransactionInterface )------------------------- */
public function getApplicationTransactionEditor() {
return new PhabricatorRepositoryEditor();
}
public function getApplicationTransactionTemplate() {
return new PhabricatorRepositoryTransaction();
}
/* -( PhabricatorPolicyInterface )----------------------------------------- */
public function getCapabilities() {
return array(
PhabricatorPolicyCapability::CAN_VIEW,
PhabricatorPolicyCapability::CAN_EDIT,
DiffusionPushCapability::CAPABILITY,
);
}
public function getPolicy($capability) {
switch ($capability) {
case PhabricatorPolicyCapability::CAN_VIEW:
return $this->getViewPolicy();
case PhabricatorPolicyCapability::CAN_EDIT:
return $this->getEditPolicy();
case DiffusionPushCapability::CAPABILITY:
return $this->getPushPolicy();
}
}
public function hasAutomaticCapability($capability, PhabricatorUser $user) {
return false;
}
/* -( PhabricatorMarkupInterface )----------------------------------------- */
public function getMarkupFieldKey($field) {
$hash = PhabricatorHash::digestForIndex($this->getMarkupText($field));
return "repo:{$hash}";
}
public function newMarkupEngine($field) {
return PhabricatorMarkupEngine::newMarkupEngine(array());
}
public function getMarkupText($field) {
return $this->getDetail('description');
}
public function didMarkupText(
$field,
$output,
PhutilMarkupEngine $engine) {
require_celerity_resource('phabricator-remarkup-css');
return phutil_tag(
'div',
array(
'class' => 'phabricator-remarkup',
),
$output);
}
public function shouldUseMarkupCache($field) {
return true;
}
/* -( PhabricatorDestructibleInterface )----------------------------------- */
public function destroyObjectPermanently(
PhabricatorDestructionEngine $engine) {
$phid = $this->getPHID();
$this->openTransaction();
$this->delete();
PhabricatorRepositoryURIIndex::updateRepositoryURIs($phid, array());
$books = id(new DivinerBookQuery())
->setViewer($engine->getViewer())
->withRepositoryPHIDs(array($phid))
->execute();
foreach ($books as $book) {
$engine->destroyObject($book);
}
$atoms = id(new DivinerAtomQuery())
->setViewer($engine->getViewer())
->withRepositoryPHIDs(array($phid))
->execute();
foreach ($atoms as $atom) {
$engine->destroyObject($atom);
}
$lfs_refs = id(new PhabricatorRepositoryGitLFSRefQuery())
->setViewer($engine->getViewer())
->withRepositoryPHIDs(array($phid))
->execute();
foreach ($lfs_refs as $ref) {
$engine->destroyObject($ref);
}
$this->saveTransaction();
}
/* -( PhabricatorDestructibleCodexInterface )------------------------------ */
public function newDestructibleCodex() {
return new PhabricatorRepositoryDestructibleCodex();
}
/* -( PhabricatorSpacesInterface )----------------------------------------- */
public function getSpacePHID() {
return $this->spacePHID;
}
/* -( PhabricatorConduitResultInterface )---------------------------------- */
public function getFieldSpecificationsForConduit() {
return array(
id(new PhabricatorConduitSearchFieldSpecification())
->setKey('name')
->setType('string')
->setDescription(pht('The repository name.')),
id(new PhabricatorConduitSearchFieldSpecification())
->setKey('vcs')
->setType('string')
->setDescription(
pht('The VCS this repository uses ("git", "hg" or "svn").')),
id(new PhabricatorConduitSearchFieldSpecification())
->setKey('callsign')
->setType('string')
->setDescription(pht('The repository callsign, if it has one.')),
id(new PhabricatorConduitSearchFieldSpecification())
->setKey('shortName')
->setType('string')
->setDescription(pht('Unique short name, if the repository has one.')),
id(new PhabricatorConduitSearchFieldSpecification())
->setKey('status')
->setType('string')
->setDescription(pht('Active or inactive status.')),
id(new PhabricatorConduitSearchFieldSpecification())
->setKey('isImporting')
->setType('bool')
->setDescription(
pht(
'True if the repository is importing initial commits.')),
id(new PhabricatorConduitSearchFieldSpecification())
->setKey('almanacServicePHID')
->setType('phid?')
->setDescription(
pht(
'The Almanac Service that hosts this repository, if the '.
'repository is clustered.')),
id(new PhabricatorConduitSearchFieldSpecification())
->setKey('refRules')
->setType('map<string, list<string>>')
->setDescription(
pht(
'The "Fetch" and "Permanent Ref" rules for this repository.')),
id(new PhabricatorConduitSearchFieldSpecification())
->setKey('defaultBranch')
->setType('string?')
->setDescription(pht('Default branch name.')),
id(new PhabricatorConduitSearchFieldSpecification())
->setKey('description')
->setType('remarkup')
->setDescription(pht('Repository description.')),
);
}
public function getFieldValuesForConduit() {
$fetch_rules = $this->getFetchRules();
$track_rules = $this->getTrackOnlyRules();
$permanent_rules = $this->getPermanentRefRules();
$fetch_rules = $this->getStringListForConduit($fetch_rules);
$track_rules = $this->getStringListForConduit($track_rules);
$permanent_rules = $this->getStringListForConduit($permanent_rules);
$default_branch = $this->getDefaultBranch();
if (!strlen($default_branch)) {
$default_branch = null;
}
return array(
'name' => $this->getName(),
'vcs' => $this->getVersionControlSystem(),
'callsign' => $this->getCallsign(),
'shortName' => $this->getRepositorySlug(),
'status' => $this->getStatus(),
'isImporting' => (bool)$this->isImporting(),
'almanacServicePHID' => $this->getAlmanacServicePHID(),
'refRules' => array(
'fetchRules' => $fetch_rules,
'trackRules' => $track_rules,
'permanentRefRules' => $permanent_rules,
),
'defaultBranch' => $default_branch,
'description' => array(
'raw' => (string)$this->getDetail('description'),
),
);
}
private function getStringListForConduit($list) {
if (!is_array($list)) {
$list = array();
}
foreach ($list as $key => $value) {
$value = (string)$value;
if (!strlen($value)) {
unset($list[$key]);
}
}
return array_values($list);
}
public function getConduitSearchAttachments() {
return array(
id(new DiffusionRepositoryURIsSearchEngineAttachment())
->setAttachmentKey('uris'),
id(new DiffusionRepositoryMetricsSearchEngineAttachment())
->setAttachmentKey('metrics'),
);
}
/* -( PhabricatorFulltextInterface )--------------------------------------- */
public function newFulltextEngine() {
return new PhabricatorRepositoryFulltextEngine();
}
/* -( PhabricatorFerretInterface )----------------------------------------- */
public function newFerretEngine() {
return new PhabricatorRepositoryFerretEngine();
}
}
diff --git a/src/infrastructure/cluster/PhabricatorDatabaseRef.php b/src/infrastructure/cluster/PhabricatorDatabaseRef.php
index 1eb232ad86..e526eb1639 100644
--- a/src/infrastructure/cluster/PhabricatorDatabaseRef.php
+++ b/src/infrastructure/cluster/PhabricatorDatabaseRef.php
@@ -1,748 +1,748 @@
<?php
final class PhabricatorDatabaseRef
extends Phobject {
const STATUS_OKAY = 'okay';
const STATUS_FAIL = 'fail';
const STATUS_AUTH = 'auth';
const STATUS_REPLICATION_CLIENT = 'replication-client';
const REPLICATION_OKAY = 'okay';
const REPLICATION_MASTER_REPLICA = 'master-replica';
const REPLICATION_REPLICA_NONE = 'replica-none';
const REPLICATION_SLOW = 'replica-slow';
const REPLICATION_NOT_REPLICATING = 'not-replicating';
const KEY_HEALTH = 'cluster.db.health';
const KEY_REFS = 'cluster.db.refs';
const KEY_INDIVIDUAL = 'cluster.db.individual';
private $host;
private $port;
private $user;
private $pass;
private $disabled;
private $isMaster;
private $isIndividual;
private $connectionLatency;
private $connectionStatus;
private $connectionMessage;
private $connectionException;
private $replicaStatus;
private $replicaMessage;
private $replicaDelay;
private $healthRecord;
private $didFailToConnect;
private $isDefaultPartition;
private $applicationMap = array();
private $masterRef;
private $replicaRefs = array();
private $usePersistentConnections;
public function setHost($host) {
$this->host = $host;
return $this;
}
public function getHost() {
return $this->host;
}
public function setPort($port) {
$this->port = $port;
return $this;
}
public function getPort() {
return $this->port;
}
public function setUser($user) {
$this->user = $user;
return $this;
}
public function getUser() {
return $this->user;
}
public function setPass(PhutilOpaqueEnvelope $pass) {
$this->pass = $pass;
return $this;
}
public function getPass() {
return $this->pass;
}
public function setIsMaster($is_master) {
$this->isMaster = $is_master;
return $this;
}
public function getIsMaster() {
return $this->isMaster;
}
public function setDisabled($disabled) {
$this->disabled = $disabled;
return $this;
}
public function getDisabled() {
return $this->disabled;
}
public function setConnectionLatency($connection_latency) {
$this->connectionLatency = $connection_latency;
return $this;
}
public function getConnectionLatency() {
return $this->connectionLatency;
}
public function setConnectionStatus($connection_status) {
$this->connectionStatus = $connection_status;
return $this;
}
public function getConnectionStatus() {
if ($this->connectionStatus === null) {
throw new PhutilInvalidStateException('queryAll');
}
return $this->connectionStatus;
}
public function setConnectionMessage($connection_message) {
$this->connectionMessage = $connection_message;
return $this;
}
public function getConnectionMessage() {
return $this->connectionMessage;
}
public function setReplicaStatus($replica_status) {
$this->replicaStatus = $replica_status;
return $this;
}
public function getReplicaStatus() {
return $this->replicaStatus;
}
public function setReplicaMessage($replica_message) {
$this->replicaMessage = $replica_message;
return $this;
}
public function getReplicaMessage() {
return $this->replicaMessage;
}
public function setReplicaDelay($replica_delay) {
$this->replicaDelay = $replica_delay;
return $this;
}
public function getReplicaDelay() {
return $this->replicaDelay;
}
public function setIsIndividual($is_individual) {
$this->isIndividual = $is_individual;
return $this;
}
public function getIsIndividual() {
return $this->isIndividual;
}
public function setIsDefaultPartition($is_default_partition) {
$this->isDefaultPartition = $is_default_partition;
return $this;
}
public function getIsDefaultPartition() {
return $this->isDefaultPartition;
}
public function setUsePersistentConnections($use_persistent_connections) {
$this->usePersistentConnections = $use_persistent_connections;
return $this;
}
public function getUsePersistentConnections() {
return $this->usePersistentConnections;
}
public function setApplicationMap(array $application_map) {
$this->applicationMap = $application_map;
return $this;
}
public function getApplicationMap() {
return $this->applicationMap;
}
public function getPartitionStateForCommit() {
$state = PhabricatorEnv::getEnvConfig('cluster.databases');
foreach ($state as $key => $value) {
// Don't store passwords, since we don't care if they differ and
// users may find it surprising.
unset($state[$key]['pass']);
}
return phutil_json_encode($state);
}
public function setMasterRef(PhabricatorDatabaseRef $master_ref) {
$this->masterRef = $master_ref;
return $this;
}
public function getMasterRef() {
return $this->masterRef;
}
public function addReplicaRef(PhabricatorDatabaseRef $replica_ref) {
$this->replicaRefs[] = $replica_ref;
return $this;
}
public function getReplicaRefs() {
return $this->replicaRefs;
}
public function getDisplayName() {
return $this->getRefKey();
}
public function getRefKey() {
$host = $this->getHost();
$port = $this->getPort();
- if (strlen($port)) {
+ if ($port) {
return "{$host}:{$port}";
}
return $host;
}
public static function getConnectionStatusMap() {
return array(
self::STATUS_OKAY => array(
'icon' => 'fa-exchange',
'color' => 'green',
'label' => pht('Okay'),
),
self::STATUS_FAIL => array(
'icon' => 'fa-times',
'color' => 'red',
'label' => pht('Failed'),
),
self::STATUS_AUTH => array(
'icon' => 'fa-key',
'color' => 'red',
'label' => pht('Invalid Credentials'),
),
self::STATUS_REPLICATION_CLIENT => array(
'icon' => 'fa-eye-slash',
'color' => 'yellow',
'label' => pht('Missing Permission'),
),
);
}
public static function getReplicaStatusMap() {
return array(
self::REPLICATION_OKAY => array(
'icon' => 'fa-download',
'color' => 'green',
'label' => pht('Okay'),
),
self::REPLICATION_MASTER_REPLICA => array(
'icon' => 'fa-database',
'color' => 'red',
'label' => pht('Replicating Master'),
),
self::REPLICATION_REPLICA_NONE => array(
'icon' => 'fa-download',
'color' => 'red',
'label' => pht('Not A Replica'),
),
self::REPLICATION_SLOW => array(
'icon' => 'fa-hourglass',
'color' => 'red',
'label' => pht('Slow Replication'),
),
self::REPLICATION_NOT_REPLICATING => array(
'icon' => 'fa-exclamation-triangle',
'color' => 'red',
'label' => pht('Not Replicating'),
),
);
}
public static function getClusterRefs() {
$cache = PhabricatorCaches::getRequestCache();
$refs = $cache->getKey(self::KEY_REFS);
if (!$refs) {
$refs = self::newRefs();
$cache->setKey(self::KEY_REFS, $refs);
}
return $refs;
}
public static function getLiveIndividualRef() {
$cache = PhabricatorCaches::getRequestCache();
$ref = $cache->getKey(self::KEY_INDIVIDUAL);
if (!$ref) {
$ref = self::newIndividualRef();
$cache->setKey(self::KEY_INDIVIDUAL, $ref);
}
return $ref;
}
public static function newRefs() {
$default_port = PhabricatorEnv::getEnvConfig('mysql.port');
$default_port = nonempty($default_port, 3306);
$default_user = PhabricatorEnv::getEnvConfig('mysql.user');
$default_pass = PhabricatorEnv::getEnvConfig('mysql.pass');
$default_pass = phutil_string_cast($default_pass);
$default_pass = new PhutilOpaqueEnvelope($default_pass);
$config = PhabricatorEnv::getEnvConfig('cluster.databases');
return id(new PhabricatorDatabaseRefParser())
->setDefaultPort($default_port)
->setDefaultUser($default_user)
->setDefaultPass($default_pass)
->newRefs($config);
}
public static function queryAll() {
$refs = self::getActiveDatabaseRefs();
return self::queryRefs($refs);
}
private static function queryRefs(array $refs) {
foreach ($refs as $ref) {
$conn = $ref->newManagementConnection();
$t_start = microtime(true);
$replica_status = false;
try {
$replica_status = queryfx_one($conn, 'SHOW SLAVE STATUS');
$ref->setConnectionStatus(self::STATUS_OKAY);
} catch (AphrontAccessDeniedQueryException $ex) {
$ref->setConnectionStatus(self::STATUS_REPLICATION_CLIENT);
$ref->setConnectionMessage(
pht(
'No permission to run "SHOW SLAVE STATUS". Grant this user '.
'"REPLICATION CLIENT" permission to allow this server to '.
'monitor replica health.'));
} catch (AphrontInvalidCredentialsQueryException $ex) {
$ref->setConnectionStatus(self::STATUS_AUTH);
$ref->setConnectionMessage($ex->getMessage());
} catch (AphrontQueryException $ex) {
$ref->setConnectionStatus(self::STATUS_FAIL);
$class = get_class($ex);
$message = $ex->getMessage();
$ref->setConnectionMessage(
pht(
'%s: %s',
get_class($ex),
$ex->getMessage()));
}
$t_end = microtime(true);
$ref->setConnectionLatency($t_end - $t_start);
if ($replica_status !== false) {
$is_replica = (bool)$replica_status;
if ($ref->getIsMaster() && $is_replica) {
$ref->setReplicaStatus(self::REPLICATION_MASTER_REPLICA);
$ref->setReplicaMessage(
pht(
'This host has a "master" role, but is replicating data from '.
'another host ("%s")!',
idx($replica_status, 'Master_Host')));
} else if (!$ref->getIsMaster() && !$is_replica) {
$ref->setReplicaStatus(self::REPLICATION_REPLICA_NONE);
$ref->setReplicaMessage(
pht(
'This host has a "replica" role, but is not replicating data '.
'from a master (no output from "SHOW SLAVE STATUS").'));
} else {
$ref->setReplicaStatus(self::REPLICATION_OKAY);
}
if ($is_replica) {
$latency = idx($replica_status, 'Seconds_Behind_Master');
if (!strlen($latency)) {
$ref->setReplicaStatus(self::REPLICATION_NOT_REPLICATING);
} else {
$latency = (int)$latency;
$ref->setReplicaDelay($latency);
if ($latency > 30) {
$ref->setReplicaStatus(self::REPLICATION_SLOW);
$ref->setReplicaMessage(
pht(
'This replica is lagging far behind the master. Data is at '.
'risk!'));
}
}
}
}
}
return $refs;
}
public function newManagementConnection() {
return $this->newConnection(
array(
'retries' => 0,
'timeout' => 2,
));
}
public function newApplicationConnection($database) {
return $this->newConnection(
array(
'database' => $database,
));
}
public function isSevered() {
// If we only have an individual database, never sever our connection to
// it, at least for now. It's possible that using the same severing rules
// might eventually make sense to help alleviate load-related failures,
// but we should wait for all the cluster stuff to stabilize first.
if ($this->getIsIndividual()) {
return false;
}
if ($this->didFailToConnect) {
return true;
}
$record = $this->getHealthRecord();
$is_healthy = $record->getIsHealthy();
if (!$is_healthy) {
return true;
}
return false;
}
public function isReachable(AphrontDatabaseConnection $connection) {
$record = $this->getHealthRecord();
$should_check = $record->getShouldCheck();
if ($this->isSevered() && !$should_check) {
return false;
}
$this->connectionException = null;
try {
$connection->openConnection();
$reachable = true;
} catch (AphrontSchemaQueryException $ex) {
// We get one of these if the database we're trying to select does not
// exist. In this case, just re-throw the exception. This is expected
// during first-time setup, when databases like "config" will not exist
// yet.
throw $ex;
} catch (Exception $ex) {
$this->connectionException = $ex;
$reachable = false;
}
if ($should_check) {
$record->didHealthCheck($reachable);
}
if (!$reachable) {
$this->didFailToConnect = true;
}
return $reachable;
}
public function checkHealth() {
$health = $this->getHealthRecord();
$should_check = $health->getShouldCheck();
if ($should_check) {
// This does an implicit health update.
$connection = $this->newManagementConnection();
$this->isReachable($connection);
}
return $this;
}
private function getHealthRecordCacheKey() {
$host = $this->getHost();
$port = $this->getPort();
$key = self::KEY_HEALTH;
return "{$key}({$host}, {$port})";
}
public function getHealthRecord() {
if (!$this->healthRecord) {
$this->healthRecord = new PhabricatorClusterServiceHealthRecord(
$this->getHealthRecordCacheKey());
}
return $this->healthRecord;
}
public function getConnectionException() {
return $this->connectionException;
}
public static function getActiveDatabaseRefs() {
$refs = array();
foreach (self::getMasterDatabaseRefs() as $ref) {
$refs[] = $ref;
}
foreach (self::getReplicaDatabaseRefs() as $ref) {
$refs[] = $ref;
}
return $refs;
}
public static function getAllMasterDatabaseRefs() {
$refs = self::getClusterRefs();
if (!$refs) {
return array(self::getLiveIndividualRef());
}
$masters = array();
foreach ($refs as $ref) {
if ($ref->getIsMaster()) {
$masters[] = $ref;
}
}
return $masters;
}
public static function getMasterDatabaseRefs() {
$refs = self::getAllMasterDatabaseRefs();
return self::getEnabledRefs($refs);
}
public function isApplicationHost($database) {
return isset($this->applicationMap[$database]);
}
public function loadRawMySQLConfigValue($key) {
$conn = $this->newManagementConnection();
try {
$value = queryfx_one($conn, 'SELECT @@%C', $key);
// NOTE: Although MySQL allows us to escape configuration values as if
// they are column names, the escaping is included in the column name
// of the return value: if we select "@@`x`", we get back a column named
// "@@`x`", not "@@x" as we might expect.
$value = head($value);
} catch (AphrontQueryException $ex) {
$value = null;
}
return $value;
}
public static function getMasterDatabaseRefForApplication($application) {
$masters = self::getMasterDatabaseRefs();
$application_master = null;
$default_master = null;
foreach ($masters as $master) {
if ($master->isApplicationHost($application)) {
$application_master = $master;
break;
}
if ($master->getIsDefaultPartition()) {
$default_master = $master;
}
}
if ($application_master) {
$masters = array($application_master);
} else if ($default_master) {
$masters = array($default_master);
} else {
$masters = array();
}
$masters = self::getEnabledRefs($masters);
$master = head($masters);
return $master;
}
public static function newIndividualRef() {
$default_user = PhabricatorEnv::getEnvConfig('mysql.user');
$default_pass = new PhutilOpaqueEnvelope(
PhabricatorEnv::getEnvConfig('mysql.pass'));
$default_host = PhabricatorEnv::getEnvConfig('mysql.host');
$default_port = PhabricatorEnv::getEnvConfig('mysql.port');
return id(new self())
->setUser($default_user)
->setPass($default_pass)
->setHost($default_host)
->setPort($default_port)
->setIsIndividual(true)
->setIsMaster(true)
->setIsDefaultPartition(true)
->setUsePersistentConnections(false);
}
public static function getAllReplicaDatabaseRefs() {
$refs = self::getClusterRefs();
if (!$refs) {
return array();
}
$replicas = array();
foreach ($refs as $ref) {
if ($ref->getIsMaster()) {
continue;
}
$replicas[] = $ref;
}
return $replicas;
}
public static function getReplicaDatabaseRefs() {
$refs = self::getAllReplicaDatabaseRefs();
return self::getEnabledRefs($refs);
}
private static function getEnabledRefs(array $refs) {
foreach ($refs as $key => $ref) {
if ($ref->getDisabled()) {
unset($refs[$key]);
}
}
return $refs;
}
public static function getReplicaDatabaseRefForApplication($application) {
$replicas = self::getReplicaDatabaseRefs();
$application_replicas = array();
$default_replicas = array();
foreach ($replicas as $replica) {
$master = $replica->getMasterRef();
if ($master->isApplicationHost($application)) {
$application_replicas[] = $replica;
}
if ($master->getIsDefaultPartition()) {
$default_replicas[] = $replica;
}
}
if ($application_replicas) {
$replicas = $application_replicas;
} else {
$replicas = $default_replicas;
}
$replicas = self::getEnabledRefs($replicas);
// TODO: We may have multiple replicas to choose from, and could make
// more of an effort to pick the "best" one here instead of always
// picking the first one. Once we've picked one, we should try to use
// the same replica for the rest of the request, though.
return head($replicas);
}
private function newConnection(array $options) {
// If we believe the database is unhealthy, don't spend as much time
// trying to connect to it, since it's likely to continue to fail and
// hammering it can only make the problem worse.
$record = $this->getHealthRecord();
if ($record->getIsHealthy()) {
$default_retries = 3;
$default_timeout = 10;
} else {
$default_retries = 0;
$default_timeout = 2;
}
$spec = $options + array(
'user' => $this->getUser(),
'pass' => $this->getPass(),
'host' => $this->getHost(),
'port' => $this->getPort(),
'database' => null,
'retries' => $default_retries,
'timeout' => $default_timeout,
'persistent' => $this->getUsePersistentConnections(),
);
$is_cli = (php_sapi_name() == 'cli');
$use_persistent = false;
if (!empty($spec['persistent']) && !$is_cli) {
$use_persistent = true;
}
unset($spec['persistent']);
$connection = self::newRawConnection($spec);
// If configured, use persistent connections. See T11672 for details.
if ($use_persistent) {
$connection->setPersistent($use_persistent);
}
// Unless this is a script running from the CLI, prevent any query from
// running for more than 30 seconds. See T10849 for details.
if (!$is_cli) {
$connection->setQueryTimeout(30);
}
return $connection;
}
public static function newRawConnection(array $options) {
if (extension_loaded('mysqli')) {
return new AphrontMySQLiDatabaseConnection($options);
} else {
return new AphrontMySQLDatabaseConnection($options);
}
}
}
diff --git a/src/infrastructure/env/PhabricatorEnv.php b/src/infrastructure/env/PhabricatorEnv.php
index ca478ed0a5..617fb57af9 100644
--- a/src/infrastructure/env/PhabricatorEnv.php
+++ b/src/infrastructure/env/PhabricatorEnv.php
@@ -1,990 +1,990 @@
<?php
/**
* Manages the execution environment configuration, exposing APIs to read
* configuration settings and other similar values that are derived directly
* from configuration settings.
*
*
* = Reading Configuration =
*
* The primary role of this class is to provide an API for reading
* Phabricator configuration, @{method:getEnvConfig}:
*
* $value = PhabricatorEnv::getEnvConfig('some.key', $default);
*
* The class also handles some URI construction based on configuration, via
* the methods @{method:getURI}, @{method:getProductionURI},
* @{method:getCDNURI}, and @{method:getDoclink}.
*
* For configuration which allows you to choose a class to be responsible for
* some functionality (e.g., which mail adapter to use to deliver email),
* @{method:newObjectFromConfig} provides a simple interface that validates
* the configured value.
*
*
* = Unit Test Support =
*
* In unit tests, you can use @{method:beginScopedEnv} to create a temporary,
* mutable environment. The method returns a scope guard object which restores
* the environment when it is destroyed. For example:
*
* public function testExample() {
* $env = PhabricatorEnv::beginScopedEnv();
* $env->overrideEnv('some.key', 'new-value-for-this-test');
*
* // Some test which depends on the value of 'some.key'.
*
* }
*
* Your changes will persist until the `$env` object leaves scope or is
* destroyed.
*
* You should //not// use this in normal code.
*
*
* @task read Reading Configuration
* @task uri URI Validation
* @task test Unit Test Support
* @task internal Internals
*/
final class PhabricatorEnv extends Phobject {
private static $sourceStack;
private static $repairSource;
private static $overrideSource;
private static $requestBaseURI;
private static $cache;
private static $localeCode;
private static $readOnly;
private static $readOnlyReason;
const READONLY_CONFIG = 'config';
const READONLY_UNREACHABLE = 'unreachable';
const READONLY_SEVERED = 'severed';
const READONLY_MASTERLESS = 'masterless';
/**
* @phutil-external-symbol class PhabricatorStartup
*/
public static function initializeWebEnvironment() {
self::initializeCommonEnvironment(false);
}
public static function initializeScriptEnvironment($config_optional) {
self::initializeCommonEnvironment($config_optional);
// NOTE: This is dangerous in general, but we know we're in a script context
// and are not vulnerable to CSRF.
AphrontWriteGuard::allowDangerousUnguardedWrites(true);
// There are several places where we log information (about errors, events,
// service calls, etc.) for analysis via DarkConsole or similar. These are
// useful for web requests, but grow unboundedly in long-running scripts and
// daemons. Discard data as it arrives in these cases.
PhutilServiceProfiler::getInstance()->enableDiscardMode();
DarkConsoleErrorLogPluginAPI::enableDiscardMode();
DarkConsoleEventPluginAPI::enableDiscardMode();
}
private static function initializeCommonEnvironment($config_optional) {
PhutilErrorHandler::initialize();
self::resetUmask();
self::buildConfigurationSourceStack($config_optional);
// Force a valid timezone. If both PHP and Phabricator configuration are
// invalid, use UTC.
$tz = self::getEnvConfig('phabricator.timezone');
if ($tz) {
@date_default_timezone_set($tz);
}
$ok = @date_default_timezone_set(date_default_timezone_get());
if (!$ok) {
date_default_timezone_set('UTC');
}
// Prepend '/support/bin' and append any paths to $PATH if we need to.
$env_path = getenv('PATH');
$phabricator_path = dirname(phutil_get_library_root('phabricator'));
$support_path = $phabricator_path.'/support/bin';
$env_path = $support_path.PATH_SEPARATOR.$env_path;
$append_dirs = self::getEnvConfig('environment.append-paths');
if (!empty($append_dirs)) {
$append_path = implode(PATH_SEPARATOR, $append_dirs);
$env_path = $env_path.PATH_SEPARATOR.$append_path;
}
putenv('PATH='.$env_path);
// Write this back into $_ENV, too, so ExecFuture picks it up when creating
// subprocess environments.
$_ENV['PATH'] = $env_path;
// If an instance identifier is defined, write it into the environment so
// it's available to subprocesses.
$instance = self::getEnvConfig('cluster.instance');
- if (strlen($instance)) {
+ if (phutil_nonempty_string($instance)) {
putenv('PHABRICATOR_INSTANCE='.$instance);
$_ENV['PHABRICATOR_INSTANCE'] = $instance;
}
PhabricatorEventEngine::initialize();
// TODO: Add a "locale.default" config option once we have some reasonable
// defaults which aren't silly nonsense.
self::setLocaleCode('en_US');
// Load the preamble utility library if we haven't already. On web
// requests this loaded earlier, but we want to load it for non-web
// requests so that unit tests can call these functions.
require_once $phabricator_path.'/support/startup/preamble-utils.php';
}
public static function beginScopedLocale($locale_code) {
return new PhabricatorLocaleScopeGuard($locale_code);
}
public static function getLocaleCode() {
return self::$localeCode;
}
public static function setLocaleCode($locale_code) {
if (!$locale_code) {
return;
}
if ($locale_code == self::$localeCode) {
return;
}
try {
$locale = PhutilLocale::loadLocale($locale_code);
$translations = PhutilTranslation::getTranslationMapForLocale(
$locale_code);
$override = self::getEnvConfig('translation.override');
if (!is_array($override)) {
$override = array();
}
PhutilTranslator::getInstance()
->setLocale($locale)
->setTranslations($override + $translations);
self::$localeCode = $locale_code;
} catch (Exception $ex) {
// Just ignore this; the user likely has an out-of-date locale code.
}
}
private static function buildConfigurationSourceStack($config_optional) {
self::dropConfigCache();
$stack = new PhabricatorConfigStackSource();
self::$sourceStack = $stack;
$default_source = id(new PhabricatorConfigDefaultSource())
->setName(pht('Global Default'));
$stack->pushSource($default_source);
$env = self::getSelectedEnvironmentName();
if ($env) {
$stack->pushSource(
id(new PhabricatorConfigFileSource($env))
->setName(pht("File '%s'", $env)));
}
$stack->pushSource(
id(new PhabricatorConfigLocalSource())
->setName(pht('Local Config')));
// If the install overrides the database adapter, we might need to load
// the database adapter class before we can push on the database config.
// This config is locked and can't be edited from the web UI anyway.
foreach (self::getEnvConfig('load-libraries') as $library) {
phutil_load_library($library);
}
// Drop any class map caches, since they will have generated without
// any classes from libraries. Without this, preflight setup checks can
// cause generation of a setup check cache that omits checks defined in
// libraries, for example.
PhutilClassMapQuery::deleteCaches();
// If custom libraries specify config options, they won't get default
// values as the Default source has already been loaded, so we get it to
// pull in all options from non-phabricator libraries now they are loaded.
$default_source->loadExternalOptions();
// If this install has site config sources, load them now.
$site_sources = id(new PhutilClassMapQuery())
->setAncestorClass('PhabricatorConfigSiteSource')
->setSortMethod('getPriority')
->execute();
foreach ($site_sources as $site_source) {
$stack->pushSource($site_source);
// If the site source did anything which reads config, throw it away
// to make sure any additional site sources get clean reads.
self::dropConfigCache();
}
$masters = PhabricatorDatabaseRef::getMasterDatabaseRefs();
if (!$masters) {
self::setReadOnly(true, self::READONLY_MASTERLESS);
} else {
// If any master is severed, we drop to readonly mode. In theory we
// could try to continue if we're only missing some applications, but
// this is very complex and we're unlikely to get it right.
foreach ($masters as $master) {
// Give severed masters one last chance to get healthy.
if ($master->isSevered()) {
$master->checkHealth();
}
if ($master->isSevered()) {
self::setReadOnly(true, self::READONLY_SEVERED);
break;
}
}
}
try {
// See T13403. If we're starting up in "config optional" mode, suppress
// messages about connection retries.
if ($config_optional) {
$database_source = @new PhabricatorConfigDatabaseSource('default');
} else {
$database_source = new PhabricatorConfigDatabaseSource('default');
}
$database_source->setName(pht('Database'));
$stack->pushSource($database_source);
} catch (AphrontSchemaQueryException $exception) {
// If the database is not available, just skip this configuration
// source. This happens during `bin/storage upgrade`, `bin/conf` before
// schema setup, etc.
} catch (PhabricatorClusterStrandedException $ex) {
// This means we can't connect to any database host. That's fine as
// long as we're running a setup script like `bin/storage`.
if (!$config_optional) {
throw $ex;
}
}
// Drop the config cache one final time to make sure we're getting clean
// reads now that we've finished building the stack.
self::dropConfigCache();
}
public static function repairConfig($key, $value) {
if (!self::$repairSource) {
self::$repairSource = id(new PhabricatorConfigDictionarySource(array()))
->setName(pht('Repaired Config'));
self::$sourceStack->pushSource(self::$repairSource);
}
self::$repairSource->setKeys(array($key => $value));
self::dropConfigCache();
}
public static function overrideConfig($key, $value) {
if (!self::$overrideSource) {
self::$overrideSource = id(new PhabricatorConfigDictionarySource(array()))
->setName(pht('Overridden Config'));
self::$sourceStack->pushSource(self::$overrideSource);
}
self::$overrideSource->setKeys(array($key => $value));
self::dropConfigCache();
}
public static function getUnrepairedEnvConfig($key, $default = null) {
foreach (self::$sourceStack->getStack() as $source) {
if ($source === self::$repairSource) {
continue;
}
$result = $source->getKeys(array($key));
if ($result) {
return $result[$key];
}
}
return $default;
}
public static function getSelectedEnvironmentName() {
$env_var = 'PHABRICATOR_ENV';
$env = idx($_SERVER, $env_var);
if (!$env) {
$env = getenv($env_var);
}
if (!$env) {
$env = idx($_ENV, $env_var);
}
if (!$env) {
$root = dirname(phutil_get_library_root('phabricator'));
$path = $root.'/conf/local/ENVIRONMENT';
if (Filesystem::pathExists($path)) {
$env = trim(Filesystem::readFile($path));
}
}
return $env;
}
/* -( Reading Configuration )---------------------------------------------- */
/**
* Get the current configuration setting for a given key.
*
* If the key is not found, then throw an Exception.
*
* @task read
*/
public static function getEnvConfig($key) {
if (!self::$sourceStack) {
throw new Exception(
pht(
'Trying to read configuration "%s" before configuration has been '.
'initialized.',
$key));
}
if (isset(self::$cache[$key])) {
return self::$cache[$key];
}
if (array_key_exists($key, self::$cache)) {
return self::$cache[$key];
}
$result = self::$sourceStack->getKeys(array($key));
if (array_key_exists($key, $result)) {
self::$cache[$key] = $result[$key];
return $result[$key];
} else {
throw new Exception(
pht(
"No config value specified for key '%s'.",
$key));
}
}
/**
* Get the current configuration setting for a given key. If the key
* does not exist, return a default value instead of throwing. This is
* primarily useful for migrations involving keys which are slated for
* removal.
*
* @task read
*/
public static function getEnvConfigIfExists($key, $default = null) {
try {
return self::getEnvConfig($key);
} catch (Exception $ex) {
return $default;
}
}
/**
* Get the fully-qualified URI for a path.
*
* @task read
*/
public static function getURI($path) {
return rtrim(self::getAnyBaseURI(), '/').$path;
}
/**
* Get the fully-qualified production URI for a path.
*
* @task read
*/
public static function getProductionURI($path) {
// If we're passed a URI which already has a domain, simply return it
// unmodified. In particular, files may have URIs which point to a CDN
// domain.
$uri = new PhutilURI($path);
if ($uri->getDomain()) {
return $path;
}
$production_domain = self::getEnvConfig('phabricator.production-uri');
if (!$production_domain) {
$production_domain = self::getAnyBaseURI();
}
return rtrim($production_domain, '/').$path;
}
public static function isSelfURI($raw_uri) {
$uri = new PhutilURI($raw_uri);
$host = $uri->getDomain();
- if (!strlen($host)) {
+ if (!phutil_nonempty_string($host)) {
return false;
}
$host = phutil_utf8_strtolower($host);
$self_map = self::getSelfURIMap();
return isset($self_map[$host]);
}
private static function getSelfURIMap() {
$self_uris = array();
$self_uris[] = self::getProductionURI('/');
$self_uris[] = self::getURI('/');
$allowed_uris = self::getEnvConfig('phabricator.allowed-uris');
foreach ($allowed_uris as $allowed_uri) {
$self_uris[] = $allowed_uri;
}
$self_map = array();
foreach ($self_uris as $self_uri) {
$host = id(new PhutilURI($self_uri))->getDomain();
- if (!strlen($host)) {
+ if (!phutil_nonempty_string($host)) {
continue;
}
$host = phutil_utf8_strtolower($host);
$self_map[$host] = $host;
}
return $self_map;
}
/**
* Get the fully-qualified production URI for a static resource path.
*
* @task read
*/
public static function getCDNURI($path) {
$alt = self::getEnvConfig('security.alternate-file-domain');
if (!$alt) {
$alt = self::getAnyBaseURI();
}
$uri = new PhutilURI($alt);
$uri->setPath($path);
return (string)$uri;
}
/**
* Get the fully-qualified production URI for a documentation resource.
*
* @task read
*/
public static function getDoclink($resource, $type = 'article') {
$params = array(
'name' => $resource,
'type' => $type,
'jump' => true,
);
$uri = new PhutilURI(
'https://we.phorge.it/diviner/find/',
$params);
return phutil_string_cast($uri);
}
/**
* Build a concrete object from a configuration key.
*
* @task read
*/
public static function newObjectFromConfig($key, $args = array()) {
$class = self::getEnvConfig($key);
return newv($class, $args);
}
public static function getAnyBaseURI() {
$base_uri = self::getEnvConfig('phabricator.base-uri');
if (!$base_uri) {
$base_uri = self::getRequestBaseURI();
}
if (!$base_uri) {
throw new Exception(
pht(
"Define '%s' in your configuration to continue.",
'phabricator.base-uri'));
}
return $base_uri;
}
public static function getRequestBaseURI() {
return self::$requestBaseURI;
}
public static function setRequestBaseURI($uri) {
self::$requestBaseURI = $uri;
}
public static function isReadOnly() {
if (self::$readOnly !== null) {
return self::$readOnly;
}
return self::getEnvConfig('cluster.read-only');
}
public static function setReadOnly($read_only, $reason) {
self::$readOnly = $read_only;
self::$readOnlyReason = $reason;
}
public static function getReadOnlyMessage() {
$reason = self::getReadOnlyReason();
switch ($reason) {
case self::READONLY_MASTERLESS:
return pht(
'This server is in read-only mode (no writable database '.
'is configured).');
case self::READONLY_UNREACHABLE:
return pht(
'This server is in read-only mode (unreachable master).');
case self::READONLY_SEVERED:
return pht(
'This server is in read-only mode (major interruption).');
}
return pht('This server is in read-only mode.');
}
public static function getReadOnlyURI() {
return urisprintf(
'/readonly/%s/',
self::getReadOnlyReason());
}
public static function getReadOnlyReason() {
if (!self::isReadOnly()) {
return null;
}
if (self::$readOnlyReason !== null) {
return self::$readOnlyReason;
}
return self::READONLY_CONFIG;
}
/* -( Unit Test Support )-------------------------------------------------- */
/**
* @task test
*/
public static function beginScopedEnv() {
return new PhabricatorScopedEnv(self::pushTestEnvironment());
}
/**
* @task test
*/
private static function pushTestEnvironment() {
self::dropConfigCache();
$source = new PhabricatorConfigDictionarySource(array());
self::$sourceStack->pushSource($source);
return spl_object_hash($source);
}
/**
* @task test
*/
public static function popTestEnvironment($key) {
self::dropConfigCache();
$source = self::$sourceStack->popSource();
$stack_key = spl_object_hash($source);
if ($stack_key !== $key) {
self::$sourceStack->pushSource($source);
throw new Exception(
pht(
'Scoped environments were destroyed in a different order than they '.
'were initialized.'));
}
}
/* -( URI Validation )----------------------------------------------------- */
/**
* Detect if a URI satisfies either @{method:isValidLocalURIForLink} or
* @{method:isValidRemoteURIForLink}, i.e. is a page on this server or the
* URI of some other resource which has a valid protocol. This rejects
* garbage URIs and URIs with protocols which do not appear in the
* `uri.allowed-protocols` configuration, notably 'javascript:' URIs.
*
* NOTE: This method is generally intended to reject URIs which it may be
* unsafe to put in an "href" link attribute.
*
* @param string URI to test.
* @return bool True if the URI identifies a web resource.
* @task uri
*/
public static function isValidURIForLink($uri) {
return self::isValidLocalURIForLink($uri) ||
self::isValidRemoteURIForLink($uri);
}
/**
* Detect if a URI identifies some page on this server.
*
* NOTE: This method is generally intended to reject URIs which it may be
* unsafe to issue a "Location:" redirect to.
*
* @param string URI to test.
* @return bool True if the URI identifies a local page.
* @task uri
*/
public static function isValidLocalURIForLink($uri) {
$uri = (string)$uri;
- if (!strlen($uri)) {
+ if (!phutil_nonempty_string($uri)) {
return false;
}
if (preg_match('/\s/', $uri)) {
// PHP hasn't been vulnerable to header injection attacks for a bunch of
// years, but we can safely reject these anyway since they're never valid.
return false;
}
// Chrome (at a minimum) interprets backslashes in Location headers and the
// URL bar as forward slashes. This is probably intended to reduce user
// error caused by confusion over which key is "forward slash" vs "back
// slash".
//
// However, it means a URI like "/\evil.com" is interpreted like
// "//evil.com", which is a protocol relative remote URI.
//
// Since we currently never generate URIs with backslashes in them, reject
// these unconditionally rather than trying to figure out how browsers will
// interpret them.
if (preg_match('/\\\\/', $uri)) {
return false;
}
// Valid URIs must begin with '/', followed by the end of the string or some
// other non-'/' character. This rejects protocol-relative URIs like
// "//evil.com/evil_stuff/".
return (bool)preg_match('@^/([^/]|$)@', $uri);
}
/**
* Detect if a URI identifies some valid linkable remote resource.
*
* @param string URI to test.
* @return bool True if a URI identifies a remote resource with an allowed
* protocol.
* @task uri
*/
public static function isValidRemoteURIForLink($uri) {
try {
self::requireValidRemoteURIForLink($uri);
return true;
} catch (Exception $ex) {
return false;
}
}
/**
* Detect if a URI identifies a valid linkable remote resource, throwing a
* detailed message if it does not.
*
* A valid linkable remote resource can be safely linked or redirected to.
* This is primarily a protocol whitelist check.
*
* @param string URI to test.
* @return void
* @task uri
*/
public static function requireValidRemoteURIForLink($raw_uri) {
$uri = new PhutilURI($raw_uri);
$proto = $uri->getProtocol();
- if (!strlen($proto)) {
+ if (!$proto) {
throw new Exception(
pht(
'URI "%s" is not a valid linkable resource. A valid linkable '.
'resource URI must specify a protocol.',
$raw_uri));
}
$protocols = self::getEnvConfig('uri.allowed-protocols');
if (!isset($protocols[$proto])) {
throw new Exception(
pht(
'URI "%s" is not a valid linkable resource. A valid linkable '.
'resource URI must use one of these protocols: %s.',
$raw_uri,
implode(', ', array_keys($protocols))));
}
$domain = $uri->getDomain();
- if (!strlen($domain)) {
+ if (!$domain) {
throw new Exception(
pht(
'URI "%s" is not a valid linkable resource. A valid linkable '.
'resource URI must specify a domain.',
$raw_uri));
}
}
/**
* Detect if a URI identifies a valid fetchable remote resource.
*
* @param string URI to test.
* @param list<string> Allowed protocols.
* @return bool True if the URI is a valid fetchable remote resource.
* @task uri
*/
public static function isValidRemoteURIForFetch($uri, array $protocols) {
try {
self::requireValidRemoteURIForFetch($uri, $protocols);
return true;
} catch (Exception $ex) {
return false;
}
}
/**
* Detect if a URI identifies a valid fetchable remote resource, throwing
* a detailed message if it does not.
*
* A valid fetchable remote resource can be safely fetched using a request
* originating on this server. This is a primarily an address check against
* the outbound address blacklist.
*
* @param string URI to test.
* @param list<string> Allowed protocols.
* @return pair<string, string> Pre-resolved URI and domain.
* @task uri
*/
public static function requireValidRemoteURIForFetch(
$raw_uri,
array $protocols) {
$uri = new PhutilURI($raw_uri);
$proto = $uri->getProtocol();
- if (!strlen($proto)) {
+ if (!$proto) {
throw new Exception(
pht(
'URI "%s" is not a valid fetchable resource. A valid fetchable '.
'resource URI must specify a protocol.',
$raw_uri));
}
$protocols = array_fuse($protocols);
if (!isset($protocols[$proto])) {
throw new Exception(
pht(
'URI "%s" is not a valid fetchable resource. A valid fetchable '.
'resource URI must use one of these protocols: %s.',
$raw_uri,
implode(', ', array_keys($protocols))));
}
$domain = $uri->getDomain();
- if (!strlen($domain)) {
+ if (!$domain) {
throw new Exception(
pht(
'URI "%s" is not a valid fetchable resource. A valid fetchable '.
'resource URI must specify a domain.',
$raw_uri));
}
$addresses = gethostbynamel($domain);
if (!$addresses) {
throw new Exception(
pht(
'URI "%s" is not a valid fetchable resource. The domain "%s" could '.
'not be resolved.',
$raw_uri,
$domain));
}
foreach ($addresses as $address) {
if (self::isBlacklistedOutboundAddress($address)) {
throw new Exception(
pht(
'URI "%s" is not a valid fetchable resource. The domain "%s" '.
'resolves to the address "%s", which is blacklisted for '.
'outbound requests.',
$raw_uri,
$domain,
$address));
}
}
$resolved_uri = clone $uri;
$resolved_uri->setDomain(head($addresses));
return array($resolved_uri, $domain);
}
/**
* Determine if an IP address is in the outbound address blacklist.
*
* @param string IP address.
* @return bool True if the address is blacklisted.
*/
public static function isBlacklistedOutboundAddress($address) {
$blacklist = self::getEnvConfig('security.outbound-blacklist');
return PhutilCIDRList::newList($blacklist)->containsAddress($address);
}
public static function isClusterRemoteAddress() {
$cluster_addresses = self::getEnvConfig('cluster.addresses');
if (!$cluster_addresses) {
return false;
}
$address = self::getRemoteAddress();
if (!$address) {
throw new Exception(
pht(
'Unable to test remote address against cluster whitelist: '.
'REMOTE_ADDR is not defined or not valid.'));
}
return self::isClusterAddress($address);
}
public static function isClusterAddress($address) {
$cluster_addresses = self::getEnvConfig('cluster.addresses');
if (!$cluster_addresses) {
throw new Exception(
pht(
'This server is not configured to serve cluster requests. '.
'Set `cluster.addresses` in the configuration to whitelist '.
'cluster hosts before sending requests that use a cluster '.
'authentication mechanism.'));
}
return PhutilCIDRList::newList($cluster_addresses)
->containsAddress($address);
}
public static function getRemoteAddress() {
$address = idx($_SERVER, 'REMOTE_ADDR');
if (!$address) {
return null;
}
try {
return PhutilIPAddress::newAddress($address);
} catch (Exception $ex) {
return null;
}
}
/* -( Internals )---------------------------------------------------------- */
/**
* @task internal
*/
public static function envConfigExists($key) {
return array_key_exists($key, self::$sourceStack->getKeys(array($key)));
}
/**
* @task internal
*/
public static function getAllConfigKeys() {
return self::$sourceStack->getAllKeys();
}
public static function getConfigSourceStack() {
return self::$sourceStack;
}
/**
* @task internal
*/
public static function overrideTestEnvConfig($stack_key, $key, $value) {
$tmp = array();
// If we don't have the right key, we'll throw when popping the last
// source off the stack.
do {
$source = self::$sourceStack->popSource();
array_unshift($tmp, $source);
if (spl_object_hash($source) == $stack_key) {
$source->setKeys(array($key => $value));
break;
}
} while (true);
foreach ($tmp as $source) {
self::$sourceStack->pushSource($source);
}
self::dropConfigCache();
}
private static function dropConfigCache() {
self::$cache = array();
}
private static function resetUmask() {
// Reset the umask to the common standard umask. The umask controls default
// permissions when files are created and propagates to subprocesses.
// "022" is the most common umask, but sometimes it is set to something
// unusual by the calling environment.
// Since various things rely on this umask to work properly and we are
// not aware of any legitimate reasons to adjust it, unconditionally
// normalize it until such reasons arise. See T7475 for discussion.
umask(022);
}
/**
* Get the path to an empty directory which is readable by all of the system
* user accounts that Phabricator acts as.
*
* In some cases, a binary needs some valid HOME or CWD to continue, but not
* all user accounts have valid home directories and even if they do they
* may not be readable after a `sudo` operation.
*
* @return string Path to an empty directory suitable for use as a CWD.
*/
public static function getEmptyCWD() {
$root = dirname(phutil_get_library_root('phabricator'));
return $root.'/support/empty/';
}
}
diff --git a/src/infrastructure/log/PhabricatorSSHLog.php b/src/infrastructure/log/PhabricatorSSHLog.php
index de4c270f72..e3c15de6bf 100644
--- a/src/infrastructure/log/PhabricatorSSHLog.php
+++ b/src/infrastructure/log/PhabricatorSSHLog.php
@@ -1,53 +1,53 @@
<?php
final class PhabricatorSSHLog extends Phobject {
private static $log;
public static function getLog() {
if (!self::$log) {
$path = PhabricatorEnv::getEnvConfig('log.ssh.path');
$format = PhabricatorEnv::getEnvConfig('log.ssh.format');
$format = nonempty(
$format,
"[%D]\t%p\t%h\t%r\t%s\t%S\t%u\t%C\t%U\t%c\t%T\t%i\t%o");
// NOTE: Path may be null. We still create the log, it just won't write
// anywhere.
$data = array(
'D' => date('r'),
'h' => php_uname('n'),
'p' => getmypid(),
'e' => time(),
'I' => PhabricatorEnv::getEnvConfig('cluster.instance'),
);
$sudo_user = PhabricatorEnv::getEnvConfig('phd.user');
- if (strlen($sudo_user)) {
+ if (phutil_nonempty_string($sudo_user)) {
$data['S'] = $sudo_user;
}
if (function_exists('posix_geteuid')) {
$system_uid = posix_geteuid();
$system_info = posix_getpwuid($system_uid);
$data['s'] = idx($system_info, 'name');
}
$client = getenv('SSH_CLIENT');
if (strlen($client)) {
$remote_address = head(explode(' ', $client));
$data['r'] = $remote_address;
}
$log = id(new PhutilDeferredLog($path, $format))
->setFailQuietly(true)
->setData($data);
self::$log = $log;
}
return self::$log;
}
}

File Metadata

Mime Type
text/x-diff
Expires
Sun, Nov 16, 12:48 AM (30 m, 9 s)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
338587
Default Alt Text
(248 KB)

Event Timeline

styx copyright 2020 sirocyl / ellie (kymkdd) / ncommander / styx contributors
Use of other names and brands is not intended to reflect a claim of, or right to, use the name or brand.
styx is an operating system using the Linux kernel and components, and "styx Linux" as a phrase does not reflect the inclusion of the Linux brand or trademark in the styx project.
Linux® is the registered trademark of Linus Torvalds in the U.S. and other countries.
Terms and Conditions (TODO: fix this!)