No OneTemporary
Actions

Size

37 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/src/infrastructure/cluster/PhabricatorDatabaseRef.php b/src/infrastructure/cluster/PhabricatorDatabaseRef.php
	index 89435b5869..478f95750b 100644
	--- a/src/infrastructure/cluster/PhabricatorDatabaseRef.php
	+++ b/src/infrastructure/cluster/PhabricatorDatabaseRef.php
	@@ -1,744 +1,747 @@
	<?php

	final class PhabricatorDatabaseRef
	extends Phobject {

	const STATUS_OKAY = 'okay';
	const STATUS_FAIL = 'fail';
	const STATUS_AUTH = 'auth';
	const STATUS_REPLICATION_CLIENT = 'replication-client';

	const REPLICATION_OKAY = 'okay';
	const REPLICATION_MASTER_REPLICA = 'master-replica';
	const REPLICATION_REPLICA_NONE = 'replica-none';
	const REPLICATION_SLOW = 'replica-slow';
	const REPLICATION_NOT_REPLICATING = 'not-replicating';

	const KEY_HEALTH = 'cluster.db.health';
	const KEY_REFS = 'cluster.db.refs';
	const KEY_INDIVIDUAL = 'cluster.db.individual';

	private $host;
	private $port;
	private $user;
	private $pass;
	private $disabled;
	private $isMaster;
	private $isIndividual;

	private $connectionLatency;
	private $connectionStatus;
	private $connectionMessage;
	private $connectionException;

	private $replicaStatus;
	private $replicaMessage;
	private $replicaDelay;

	private $healthRecord;
	private $didFailToConnect;

	private $isDefaultPartition;
	private $applicationMap = array();
	private $masterRef;
	private $replicaRefs = array();
	private $usePersistentConnections;

	public function setHost($host) {
	$this->host = $host;
	return $this;
	}

	public function getHost() {
	return $this->host;
	}

	public function setPort($port) {
	$this->port = $port;
	return $this;
	}

	public function getPort() {
	return $this->port;
	}

	public function setUser($user) {
	$this->user = $user;
	return $this;
	}

	public function getUser() {
	return $this->user;
	}

	public function setPass(PhutilOpaqueEnvelope $pass) {
	$this->pass = $pass;
	return $this;
	}

	public function getPass() {
	return $this->pass;
	}

	public function setIsMaster($is_master) {
	$this->isMaster = $is_master;
	return $this;
	}

	public function getIsMaster() {
	return $this->isMaster;
	}

	public function setDisabled($disabled) {
	$this->disabled = $disabled;
	return $this;
	}

	public function getDisabled() {
	return $this->disabled;
	}

	public function setConnectionLatency($connection_latency) {
	$this->connectionLatency = $connection_latency;
	return $this;
	}

	public function getConnectionLatency() {
	return $this->connectionLatency;
	}

	public function setConnectionStatus($connection_status) {
	$this->connectionStatus = $connection_status;
	return $this;
	}

	public function getConnectionStatus() {
	if ($this->connectionStatus === null) {
	throw new PhutilInvalidStateException('queryAll');
	}

	return $this->connectionStatus;
	}

	public function setConnectionMessage($connection_message) {
	$this->connectionMessage = $connection_message;
	return $this;
	}

	public function getConnectionMessage() {
	return $this->connectionMessage;
	}

	public function setReplicaStatus($replica_status) {
	$this->replicaStatus = $replica_status;
	return $this;
	}

	public function getReplicaStatus() {
	return $this->replicaStatus;
	}

	public function setReplicaMessage($replica_message) {
	$this->replicaMessage = $replica_message;
	return $this;
	}

	public function getReplicaMessage() {
	return $this->replicaMessage;
	}

	public function setReplicaDelay($replica_delay) {
	$this->replicaDelay = $replica_delay;
	return $this;
	}

	public function getReplicaDelay() {
	return $this->replicaDelay;
	}

	public function setIsIndividual($is_individual) {
	$this->isIndividual = $is_individual;
	return $this;
	}

	public function getIsIndividual() {
	return $this->isIndividual;
	}

	public function setIsDefaultPartition($is_default_partition) {
	$this->isDefaultPartition = $is_default_partition;
	return $this;
	}

	public function getIsDefaultPartition() {
	return $this->isDefaultPartition;
	}

	public function setUsePersistentConnections($use_persistent_connections) {
	$this->usePersistentConnections = $use_persistent_connections;
	return $this;
	}

	public function getUsePersistentConnections() {
	return $this->usePersistentConnections;
	}

	public function setApplicationMap(array $application_map) {
	$this->applicationMap = $application_map;
	return $this;
	}

	public function getApplicationMap() {
	return $this->applicationMap;
	}

	public function getPartitionStateForCommit() {
	$state = PhabricatorEnv::getEnvConfig('cluster.databases');
	foreach ($state as $key => $value) {
	// Don't store passwords, since we don't care if they differ and
	// users may find it surprising.
	unset($state[$key]['pass']);
	}

	return phutil_json_encode($state);
	}

	public function setMasterRef(PhabricatorDatabaseRef $master_ref) {
	$this->masterRef = $master_ref;
	return $this;
	}

	public function getMasterRef() {
	return $this->masterRef;
	}

	public function addReplicaRef(PhabricatorDatabaseRef $replica_ref) {
	$this->replicaRefs[] = $replica_ref;
	return $this;
	}

	public function getReplicaRefs() {
	return $this->replicaRefs;
	}

	+ public function getDisplayName() {
	+ return $this->getRefKey();
	+ }

	public function getRefKey() {
	$host = $this->getHost();

	$port = $this->getPort();
	if (strlen($port)) {
	return "{$host}:{$port}";
	}

	return $host;
	}

	public static function getConnectionStatusMap() {
	return array(
	self::STATUS_OKAY => array(
	'icon' => 'fa-exchange',
	'color' => 'green',
	'label' => pht('Okay'),
	),
	self::STATUS_FAIL => array(
	'icon' => 'fa-times',
	'color' => 'red',
	'label' => pht('Failed'),
	),
	self::STATUS_AUTH => array(
	'icon' => 'fa-key',
	'color' => 'red',
	'label' => pht('Invalid Credentials'),
	),
	self::STATUS_REPLICATION_CLIENT => array(
	'icon' => 'fa-eye-slash',
	'color' => 'yellow',
	'label' => pht('Missing Permission'),
	),
	);
	}

	public static function getReplicaStatusMap() {
	return array(
	self::REPLICATION_OKAY => array(
	'icon' => 'fa-download',
	'color' => 'green',
	'label' => pht('Okay'),
	),
	self::REPLICATION_MASTER_REPLICA => array(
	'icon' => 'fa-database',
	'color' => 'red',
	'label' => pht('Replicating Master'),
	),
	self::REPLICATION_REPLICA_NONE => array(
	'icon' => 'fa-download',
	'color' => 'red',
	'label' => pht('Not A Replica'),
	),
	self::REPLICATION_SLOW => array(
	'icon' => 'fa-hourglass',
	'color' => 'red',
	'label' => pht('Slow Replication'),
	),
	self::REPLICATION_NOT_REPLICATING => array(
	'icon' => 'fa-exclamation-triangle',
	'color' => 'red',
	'label' => pht('Not Replicating'),
	),
	);
	}

	public static function getClusterRefs() {
	$cache = PhabricatorCaches::getRequestCache();

	$refs = $cache->getKey(self::KEY_REFS);
	if (!$refs) {
	$refs = self::newRefs();
	$cache->setKey(self::KEY_REFS, $refs);
	}

	return $refs;
	}

	public static function getLiveIndividualRef() {
	$cache = PhabricatorCaches::getRequestCache();

	$ref = $cache->getKey(self::KEY_INDIVIDUAL);
	if (!$ref) {
	$ref = self::newIndividualRef();
	$cache->setKey(self::KEY_INDIVIDUAL, $ref);
	}

	return $ref;
	}

	public static function newRefs() {
	$default_port = PhabricatorEnv::getEnvConfig('mysql.port');
	$default_port = nonempty($default_port, 3306);

	$default_user = PhabricatorEnv::getEnvConfig('mysql.user');

	$default_pass = PhabricatorEnv::getEnvConfig('mysql.pass');
	$default_pass = new PhutilOpaqueEnvelope($default_pass);

	$config = PhabricatorEnv::getEnvConfig('cluster.databases');

	return id(new PhabricatorDatabaseRefParser())
	->setDefaultPort($default_port)
	->setDefaultUser($default_user)
	->setDefaultPass($default_pass)
	->newRefs($config);
	}

	public static function queryAll() {
	$refs = self::getActiveDatabaseRefs();
	return self::queryRefs($refs);
	}

	private static function queryRefs(array $refs) {
	foreach ($refs as $ref) {
	$conn = $ref->newManagementConnection();

	$t_start = microtime(true);
	$replica_status = false;
	try {
	$replica_status = queryfx_one($conn, 'SHOW SLAVE STATUS');
	$ref->setConnectionStatus(self::STATUS_OKAY);
	} catch (AphrontAccessDeniedQueryException $ex) {
	$ref->setConnectionStatus(self::STATUS_REPLICATION_CLIENT);
	$ref->setConnectionMessage(
	pht(
	'No permission to run "SHOW SLAVE STATUS". Grant this user '.
	'"REPLICATION CLIENT" permission to allow Phabricator to '.
	'monitor replica health.'));
	} catch (AphrontInvalidCredentialsQueryException $ex) {
	$ref->setConnectionStatus(self::STATUS_AUTH);
	$ref->setConnectionMessage($ex->getMessage());
	} catch (AphrontQueryException $ex) {
	$ref->setConnectionStatus(self::STATUS_FAIL);

	$class = get_class($ex);
	$message = $ex->getMessage();
	$ref->setConnectionMessage(
	pht(
	'%s: %s',
	get_class($ex),
	$ex->getMessage()));
	}
	$t_end = microtime(true);
	$ref->setConnectionLatency($t_end - $t_start);

	if ($replica_status !== false) {
	$is_replica = (bool)$replica_status;
	if ($ref->getIsMaster() && $is_replica) {
	$ref->setReplicaStatus(self::REPLICATION_MASTER_REPLICA);
	$ref->setReplicaMessage(
	pht(
	'This host has a "master" role, but is replicating data from '.
	'another host ("%s")!',
	idx($replica_status, 'Master_Host')));
	} else if (!$ref->getIsMaster() && !$is_replica) {
	$ref->setReplicaStatus(self::REPLICATION_REPLICA_NONE);
	$ref->setReplicaMessage(
	pht(
	'This host has a "replica" role, but is not replicating data '.
	'from a master (no output from "SHOW SLAVE STATUS").'));
	} else {
	$ref->setReplicaStatus(self::REPLICATION_OKAY);
	}

	if ($is_replica) {
	$latency = idx($replica_status, 'Seconds_Behind_Master');
	if (!strlen($latency)) {
	$ref->setReplicaStatus(self::REPLICATION_NOT_REPLICATING);
	} else {
	$latency = (int)$latency;
	$ref->setReplicaDelay($latency);
	if ($latency > 30) {
	$ref->setReplicaStatus(self::REPLICATION_SLOW);
	$ref->setReplicaMessage(
	pht(
	'This replica is lagging far behind the master. Data is at '.
	'risk!'));
	}
	}
	}
	}
	}

	return $refs;
	}

	public function newManagementConnection() {
	return $this->newConnection(
	array(
	'retries' => 0,
	'timeout' => 2,
	));
	}

	public function newApplicationConnection($database) {
	return $this->newConnection(
	array(
	'database' => $database,
	));
	}

	public function isSevered() {
	// If we only have an individual database, never sever our connection to
	// it, at least for now. It's possible that using the same severing rules
	// might eventually make sense to help alleviate load-related failures,
	// but we should wait for all the cluster stuff to stabilize first.
	if ($this->getIsIndividual()) {
	return false;
	}

	if ($this->didFailToConnect) {
	return true;
	}

	$record = $this->getHealthRecord();
	$is_healthy = $record->getIsHealthy();
	if (!$is_healthy) {
	return true;
	}

	return false;
	}

	public function isReachable(AphrontDatabaseConnection $connection) {
	$record = $this->getHealthRecord();
	$should_check = $record->getShouldCheck();

	if ($this->isSevered() && !$should_check) {
	return false;
	}

	$this->connectionException = null;
	try {
	$connection->openConnection();
	$reachable = true;
	} catch (AphrontSchemaQueryException $ex) {
	// We get one of these if the database we're trying to select does not
	// exist. In this case, just re-throw the exception. This is expected
	// during first-time setup, when databases like "config" will not exist
	// yet.
	throw $ex;
	} catch (Exception $ex) {
	$this->connectionException = $ex;
	$reachable = false;
	}

	if ($should_check) {
	$record->didHealthCheck($reachable);
	}

	if (!$reachable) {
	$this->didFailToConnect = true;
	}

	return $reachable;
	}

	public function checkHealth() {
	$health = $this->getHealthRecord();

	$should_check = $health->getShouldCheck();
	if ($should_check) {
	// This does an implicit health update.
	$connection = $this->newManagementConnection();
	$this->isReachable($connection);
	}

	return $this;
	}

	private function getHealthRecordCacheKey() {
	$host = $this->getHost();
	$port = $this->getPort();
	$key = self::KEY_HEALTH;

	return "{$key}({$host}, {$port})";
	}

	public function getHealthRecord() {
	if (!$this->healthRecord) {
	$this->healthRecord = new PhabricatorClusterServiceHealthRecord(
	$this->getHealthRecordCacheKey());
	}
	return $this->healthRecord;
	}

	public function getConnectionException() {
	return $this->connectionException;
	}

	public static function getActiveDatabaseRefs() {
	$refs = array();

	foreach (self::getMasterDatabaseRefs() as $ref) {
	$refs[] = $ref;
	}

	foreach (self::getReplicaDatabaseRefs() as $ref) {
	$refs[] = $ref;
	}

	return $refs;
	}

	public static function getAllMasterDatabaseRefs() {
	$refs = self::getClusterRefs();

	if (!$refs) {
	return array(self::getLiveIndividualRef());
	}

	$masters = array();
	foreach ($refs as $ref) {
	if ($ref->getIsMaster()) {
	$masters[] = $ref;
	}
	}

	return $masters;
	}

	public static function getMasterDatabaseRefs() {
	$refs = self::getAllMasterDatabaseRefs();
	return self::getEnabledRefs($refs);
	}

	public function isApplicationHost($database) {
	return isset($this->applicationMap[$database]);
	}

	public function loadRawMySQLConfigValue($key) {
	$conn = $this->newManagementConnection();

	try {
	$value = queryfx_one($conn, 'SELECT @@%C', $key);

	// NOTE: Although MySQL allows us to escape configuration values as if
	// they are column names, the escaping is included in the column name
	// of the return value: if we select "@@`x`", we get back a column named
	// "@@`x`", not "@@x" as we might expect.
	$value = head($value);

	} catch (AphrontQueryException $ex) {
	$value = null;
	}

	return $value;
	}

	public static function getMasterDatabaseRefForApplication($application) {
	$masters = self::getMasterDatabaseRefs();

	$application_master = null;
	$default_master = null;
	foreach ($masters as $master) {
	if ($master->isApplicationHost($application)) {
	$application_master = $master;
	break;
	}
	if ($master->getIsDefaultPartition()) {
	$default_master = $master;
	}
	}

	if ($application_master) {
	$masters = array($application_master);
	} else if ($default_master) {
	$masters = array($default_master);
	} else {
	$masters = array();
	}

	$masters = self::getEnabledRefs($masters);
	$master = head($masters);

	return $master;
	}

	public static function newIndividualRef() {
	$default_user = PhabricatorEnv::getEnvConfig('mysql.user');
	$default_pass = new PhutilOpaqueEnvelope(
	PhabricatorEnv::getEnvConfig('mysql.pass'));
	$default_host = PhabricatorEnv::getEnvConfig('mysql.host');
	$default_port = PhabricatorEnv::getEnvConfig('mysql.port');

	return id(new self())
	->setUser($default_user)
	->setPass($default_pass)
	->setHost($default_host)
	->setPort($default_port)
	->setIsIndividual(true)
	->setIsMaster(true)
	->setIsDefaultPartition(true)
	->setUsePersistentConnections(false);
	}

	public static function getAllReplicaDatabaseRefs() {
	$refs = self::getClusterRefs();

	if (!$refs) {
	return array();
	}

	$replicas = array();
	foreach ($refs as $ref) {
	if ($ref->getIsMaster()) {
	continue;
	}

	$replicas[] = $ref;
	}

	return $replicas;
	}

	public static function getReplicaDatabaseRefs() {
	$refs = self::getAllReplicaDatabaseRefs();
	return self::getEnabledRefs($refs);
	}

	private static function getEnabledRefs(array $refs) {
	foreach ($refs as $key => $ref) {
	if ($ref->getDisabled()) {
	unset($refs[$key]);
	}
	}
	return $refs;
	}

	public static function getReplicaDatabaseRefForApplication($application) {
	$replicas = self::getReplicaDatabaseRefs();

	$application_replicas = array();
	$default_replicas = array();
	foreach ($replicas as $replica) {
	$master = $replica->getMasterRef();

	if ($master->isApplicationHost($application)) {
	$application_replicas[] = $replica;
	}

	if ($master->getIsDefaultPartition()) {
	$default_replicas[] = $replica;
	}
	}

	if ($application_replicas) {
	$replicas = $application_replicas;
	} else {
	$replicas = $default_replicas;
	}

	$replicas = self::getEnabledRefs($replicas);

	// TODO: We may have multiple replicas to choose from, and could make
	// more of an effort to pick the "best" one here instead of always
	// picking the first one. Once we've picked one, we should try to use
	// the same replica for the rest of the request, though.

	return head($replicas);
	}

	private function newConnection(array $options) {
	// If we believe the database is unhealthy, don't spend as much time
	// trying to connect to it, since it's likely to continue to fail and
	// hammering it can only make the problem worse.
	$record = $this->getHealthRecord();
	if ($record->getIsHealthy()) {
	$default_retries = 3;
	$default_timeout = 10;
	} else {
	$default_retries = 0;
	$default_timeout = 2;
	}

	$spec = $options + array(
	'user' => $this->getUser(),
	'pass' => $this->getPass(),
	'host' => $this->getHost(),
	'port' => $this->getPort(),
	'database' => null,
	'retries' => $default_retries,
	'timeout' => $default_timeout,
	'persistent' => $this->getUsePersistentConnections(),
	);

	$is_cli = (php_sapi_name() == 'cli');

	$use_persistent = false;
	if (!empty($spec['persistent']) && !$is_cli) {
	$use_persistent = true;
	}
	unset($spec['persistent']);

	$connection = self::newRawConnection($spec);

	// If configured, use persistent connections. See T11672 for details.
	if ($use_persistent) {
	$connection->setPersistent($use_persistent);
	}

	// Unless this is a script running from the CLI, prevent any query from
	// running for more than 30 seconds. See T10849 for details.
	if (!$is_cli) {
	$connection->setQueryTimeout(30);
	}

	return $connection;
	}

	public static function newRawConnection(array $options) {
	if (extension_loaded('mysqli')) {
	return new AphrontMySQLiDatabaseConnection($options);
	} else {
	return new AphrontMySQLDatabaseConnection($options);
	}
	}

	}
	diff --git a/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php b/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php
	index b838c8a5d9..a6a0d74593 100644
	--- a/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php
	+++ b/src/infrastructure/storage/management/PhabricatorStorageManagementAPI.php
	@@ -1,373 +1,377 @@
	<?php

	final class PhabricatorStorageManagementAPI extends Phobject {

	private $ref;
	private $host;
	private $user;
	private $port;
	private $password;
	private $namespace;
	private $conns = array();
	private $disableUTF8MB4;

	const CHARSET_DEFAULT = 'CHARSET';
	const CHARSET_SORT = 'CHARSET_SORT';
	const CHARSET_FULLTEXT = 'CHARSET_FULLTEXT';
	const COLLATE_TEXT = 'COLLATE_TEXT';
	const COLLATE_SORT = 'COLLATE_SORT';
	const COLLATE_FULLTEXT = 'COLLATE_FULLTEXT';

	const TABLE_STATUS = 'patch_status';
	const TABLE_HOSTSTATE = 'hoststate';

	public function setDisableUTF8MB4($disable_utf8_mb4) {
	$this->disableUTF8MB4 = $disable_utf8_mb4;
	return $this;
	}

	public function getDisableUTF8MB4() {
	return $this->disableUTF8MB4;
	}

	public function setNamespace($namespace) {
	$this->namespace = $namespace;
	PhabricatorLiskDAO::pushStorageNamespace($namespace);
	return $this;
	}

	public function getNamespace() {
	return $this->namespace;
	}

	public function setUser($user) {
	$this->user = $user;
	return $this;
	}

	public function getUser() {
	return $this->user;
	}

	public function setPassword($password) {
	$this->password = $password;
	return $this;
	}

	public function getPassword() {
	return $this->password;
	}

	public function setHost($host) {
	$this->host = $host;
	return $this;
	}

	public function getHost() {
	return $this->host;
	}

	public function setPort($port) {
	$this->port = $port;
	return $this;
	}

	public function getPort() {
	return $this->port;
	}

	public function setRef(PhabricatorDatabaseRef $ref) {
	$this->ref = $ref;
	return $this;
	}

	public function getRef() {
	return $this->ref;
	}

	public function getDatabaseName($fragment) {
	return $this->namespace.'_'.$fragment;
	}

	+ public function getDisplayName() {
	+ return $this->getRef()->getDisplayName();
	+ }
	+
	public function getDatabaseList(array $patches, $only_living = false) {
	assert_instances_of($patches, 'PhabricatorStoragePatch');

	$list = array();

	foreach ($patches as $patch) {
	if ($patch->getType() == 'db') {
	if ($only_living && $patch->isDead()) {
	continue;
	}
	$list[] = $this->getDatabaseName($patch->getName());
	}
	}

	return $list;
	}

	public function getConn($fragment) {
	$database = $this->getDatabaseName($fragment);
	$return = &$this->conns[$this->host][$this->user][$database];
	if (!$return) {
	$return = PhabricatorDatabaseRef::newRawConnection(
	array(
	'user' => $this->user,
	'pass' => $this->password,
	'host' => $this->host,
	'port' => $this->port,
	'database' => $fragment
	? $database
	: null,
	));
	}
	return $return;
	}

	public function getAppliedPatches() {
	try {
	$applied = queryfx_all(
	$this->getConn('meta_data'),
	'SELECT patch FROM %T',
	self::TABLE_STATUS);
	return ipull($applied, 'patch');
	} catch (AphrontAccessDeniedQueryException $ex) {
	throw new PhutilProxyException(
	pht(
	'Failed while trying to read schema status: the database "%s" '.
	'exists, but the current user ("%s") does not have permission to '.
	'access it. GRANT the current user more permissions, or use a '.
	'different user.',
	$this->getDatabaseName('meta_data'),
	$this->getUser()),
	$ex);
	} catch (AphrontQueryException $ex) {
	return null;
	}
	}

	public function getPatchDurations() {
	try {
	$rows = queryfx_all(
	$this->getConn('meta_data'),
	'SELECT patch, duration FROM %T WHERE duration IS NOT NULL',
	self::TABLE_STATUS);
	return ipull($rows, 'duration', 'patch');
	} catch (AphrontQueryException $ex) {
	return array();
	}
	}

	public function createDatabase($fragment) {
	$info = $this->getCharsetInfo();

	queryfx(
	$this->getConn(null),
	'CREATE DATABASE IF NOT EXISTS %T COLLATE %T',
	$this->getDatabaseName($fragment),
	$info[self::COLLATE_TEXT]);
	}

	public function createTable($fragment, $table, array $cols) {
	queryfx(
	$this->getConn($fragment),
	'CREATE TABLE IF NOT EXISTS %T.%T (%Q) '.
	'ENGINE=InnoDB, COLLATE utf8_general_ci',
	$this->getDatabaseName($fragment),
	$table,
	implode(', ', $cols));
	}

	public function getLegacyPatches(array $patches) {
	assert_instances_of($patches, 'PhabricatorStoragePatch');

	try {
	$row = queryfx_one(
	$this->getConn('meta_data'),
	'SELECT version FROM %T',
	'schema_version');
	$version = $row['version'];
	} catch (AphrontQueryException $ex) {
	return array();
	}

	$legacy = array();
	foreach ($patches as $key => $patch) {
	if ($patch->getLegacy() !== false && $patch->getLegacy() <= $version) {
	$legacy[] = $key;
	}
	}

	return $legacy;
	}

	public function markPatchApplied($patch, $duration = null) {
	$conn = $this->getConn('meta_data');

	queryfx(
	$conn,
	'INSERT INTO %T (patch, applied) VALUES (%s, %d)',
	self::TABLE_STATUS,
	$patch,
	time());

	// We didn't add this column for a long time, so it may not exist yet.
	if ($duration !== null) {
	try {
	queryfx(
	$conn,
	'UPDATE %T SET duration = %d WHERE patch = %s',
	self::TABLE_STATUS,
	(int)floor($duration * 1000000),
	$patch);
	} catch (AphrontQueryException $ex) {
	// Just ignore this, as it almost certainly indicates that we just
	// don't have the column yet.
	}
	}
	}

	public function applyPatch(PhabricatorStoragePatch $patch) {
	$type = $patch->getType();
	$name = $patch->getName();
	switch ($type) {
	case 'db':
	$this->createDatabase($name);
	break;
	case 'sql':
	$this->applyPatchSQL($name);
	break;
	case 'php':
	$this->applyPatchPHP($name);
	break;
	default:
	throw new Exception(pht("Unable to apply patch of type '%s'.", $type));
	}
	}

	public function applyPatchSQL($sql) {
	$sql = Filesystem::readFile($sql);
	$queries = preg_split('/;\s+/', $sql);
	$queries = array_filter($queries);

	$conn = $this->getConn(null);

	$charset_info = $this->getCharsetInfo();
	foreach ($charset_info as $key => $value) {
	$charset_info[$key] = qsprintf($conn, '%T', $value);
	}

	foreach ($queries as $query) {
	$query = str_replace('{$NAMESPACE}', $this->namespace, $query);

	foreach ($charset_info as $key => $value) {
	$query = str_replace('{$'.$key.'}', $value, $query);
	}

	try {
	// NOTE: We're using the unsafe "%Z" conversion here. There's no
	// avoiding it since we're executing raw text files full of SQL.
	queryfx($conn, '%Z', $query);
	} catch (AphrontAccessDeniedQueryException $ex) {
	throw new PhutilProxyException(
	pht(
	'Unable to access a required database or table. This almost '.
	'always means that the user you are connecting with ("%s") does '.
	'not have sufficient permissions granted in MySQL. You can '.
	'use `bin/storage databases` to get a list of all databases '.
	'permission is required on.',
	$this->getUser()),
	$ex);
	}
	}
	}

	public function applyPatchPHP($script) {
	$schema_conn = $this->getConn(null);
	require_once $script;
	}

	public function isCharacterSetAvailable($character_set) {
	if ($character_set == 'utf8mb4') {
	if ($this->getDisableUTF8MB4()) {
	return false;
	}
	}

	$conn = $this->getConn(null);
	return self::isCharacterSetAvailableOnConnection($character_set, $conn);
	}

	public function getClientCharset() {
	if ($this->isCharacterSetAvailable('utf8mb4')) {
	return 'utf8mb4';
	} else {
	return 'utf8';
	}
	}

	public static function isCharacterSetAvailableOnConnection(
	$character_set,
	AphrontDatabaseConnection $conn) {
	$result = queryfx_one(
	$conn,
	'SELECT CHARACTER_SET_NAME FROM INFORMATION_SCHEMA.CHARACTER_SETS
	WHERE CHARACTER_SET_NAME = %s',
	$character_set);

	return (bool)$result;
	}

	public function getCharsetInfo() {
	if ($this->isCharacterSetAvailable('utf8mb4')) {
	// If utf8mb4 is available, we use it with the utf8mb4_unicode_ci
	// collation. This is most correct, and will sort properly.

	$charset = 'utf8mb4';
	$charset_sort = 'utf8mb4';
	$charset_full = 'utf8mb4';
	$collate_text = 'utf8mb4_bin';
	$collate_sort = 'utf8mb4_unicode_ci';
	$collate_full = 'utf8mb4_unicode_ci';
	} else {
	// If utf8mb4 is not available, we use binary for most data. This allows
	// us to store 4-byte unicode characters.
	//
	// It's possible that strings will be truncated in the middle of a
	// character on insert. We encourage users to set STRICT_ALL_TABLES
	// to prevent this.
	//
	// For "fulltext" and "sort" columns, we don't use binary.
	//
	// With "fulltext", we can not use binary because MySQL won't let us.
	// We use 3-byte utf8 instead and accept being unable to index 4-byte
	// characters.
	//
	// With "sort", if we use binary we lose case insensitivity (for
	// example, "ALincoln@logcabin.com" and "alincoln@logcabin.com" would no
	// longer be identified as the same email address). This can be very
	// confusing and is far worse overall than not supporting 4-byte unicode
	// characters, so we use 3-byte utf8 and accept limited 4-byte support as
	// a tradeoff to get sensible collation behavior. Many columns where
	// collation is important rarely contain 4-byte characters anyway, so we
	// are not giving up too much.

	$charset = 'binary';
	$charset_sort = 'utf8';
	$charset_full = 'utf8';
	$collate_text = 'binary';
	$collate_sort = 'utf8_general_ci';
	$collate_full = 'utf8_general_ci';
	}

	return array(
	self::CHARSET_DEFAULT => $charset,
	self::CHARSET_SORT => $charset_sort,
	self::CHARSET_FULLTEXT => $charset_full,
	self::COLLATE_TEXT => $collate_text,
	self::COLLATE_SORT => $collate_sort,
	self::COLLATE_FULLTEXT => $collate_full,
	);
	}

	}
	diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php
	index 7d0946c8c3..9b718e231d 100644
	--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php
	+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php
	@@ -1,107 +1,133 @@
	<?php

	final class PhabricatorStorageManagementDestroyWorkflow
	extends PhabricatorStorageManagementWorkflow {

	protected function didConstruct() {
	$this
	->setName('destroy')
	->setExamples('destroy [__options__]')
	->setSynopsis(pht('Permanently destroy all storage and data.'))
	->setArguments(
	array(
	array(
	'name' => 'unittest-fixtures',
	'help' => pht(
	'Restrict destroy operations to databases created '.
	'by %s test fixtures.',
	'PhabricatorTestCase'),
	),
	));
	}

	public function didExecute(PhutilArgumentParser $args) {
	- $console = PhutilConsole::getConsole();
	+ $api = $this->getSingleAPI();
	+
	+ $host_display = $api->getDisplayName();

	if (!$this->isDryRun() && !$this->isForce()) {
	if ($args->getArg('unittest-fixtures')) {
	- $console->writeOut(
	- phutil_console_wrap(
	- pht(
	- 'Are you completely sure you really want to destroy all unit '.
	- 'test fixure data? This operation can not be undone.')));
	+ $warning = pht(
	+ 'Are you completely sure you really want to destroy all unit '.
	+ 'test fixure data on host "%s"? This operation can not be undone.',
	+ $host_display);
	+
	+ echo tsprintf(
	+ '%B',
	+ id(new PhutilConsoleBlock())
	+ ->addParagraph($warning)
	+ ->drawConsoleString());
	+
	if (!phutil_console_confirm(pht('Destroy all unit test data?'))) {
	- $console->writeOut("%s\n", pht('Cancelled.'));
	+ $this->logFail(
	+ pht('CANCELLED'),
	+ pht('User cancelled operation.'));
	exit(1);
	}
	} else {
	- $console->writeOut(
	- phutil_console_wrap(
	- pht(
	- 'Are you completely sure you really want to permanently destroy '.
	- 'all storage for Phabricator data? This operation can not be '.
	- 'undone and your data will not be recoverable if you proceed.')));
	+ $warning = pht(
	+ 'Are you completely sure you really want to permanently destroy '.
	+ 'all storage for Phabricator data on host "%s"? This operation '.
	+ 'can not be undone and your data will not be recoverable if '.
	+ 'you proceed.',
	+ $host_display);
	+
	+ echo tsprintf(
	+ '%B',
	+ id(new PhutilConsoleBlock())
	+ ->addParagraph($warning)
	+ ->drawConsoleString());

	if (!phutil_console_confirm(pht('Permanently destroy all data?'))) {
	- $console->writeOut("%s\n", pht('Cancelled.'));
	+ $this->logFail(
	+ pht('CANCELLED'),
	+ pht('User cancelled operation.'));
	exit(1);
	}

	if (!phutil_console_confirm(pht('Really destroy all data forever?'))) {
	- $console->writeOut("%s\n", pht('Cancelled.'));
	+ $this->logFail(
	+ pht('CANCELLED'),
	+ pht('User cancelled operation.'));
	exit(1);
	}
	}
	}

	- $apis = $this->getMasterAPIs();
	- foreach ($apis as $api) {
	- $patches = $this->getPatches();
	+ $patches = $this->getPatches();

	- if ($args->getArg('unittest-fixtures')) {
	- $conn = $api->getConn(null);
	- $databases = queryfx_all(
	- $conn,
	- 'SELECT DISTINCT(TABLE_SCHEMA) AS db '.
	- 'FROM INFORMATION_SCHEMA.TABLES '.
	- 'WHERE TABLE_SCHEMA LIKE %>',
	- PhabricatorTestCase::NAMESPACE_PREFIX);
	- $databases = ipull($databases, 'db');
	- } else {
	- $databases = $api->getDatabaseList($patches);
	- $databases[] = $api->getDatabaseName('meta_data');
	+ if ($args->getArg('unittest-fixtures')) {
	+ $conn = $api->getConn(null);
	+ $databases = queryfx_all(
	+ $conn,
	+ 'SELECT DISTINCT(TABLE_SCHEMA) AS db '.
	+ 'FROM INFORMATION_SCHEMA.TABLES '.
	+ 'WHERE TABLE_SCHEMA LIKE %>',
	+ PhabricatorTestCase::NAMESPACE_PREFIX);
	+ $databases = ipull($databases, 'db');
	+ } else {
	+ $databases = $api->getDatabaseList($patches);
	+ $databases[] = $api->getDatabaseName('meta_data');

	- // These are legacy databases that were dropped long ago. See T2237.
	- $databases[] = $api->getDatabaseName('phid');
	- $databases[] = $api->getDatabaseName('directory');
	- }
	+ // These are legacy databases that were dropped long ago. See T2237.
	+ $databases[] = $api->getDatabaseName('phid');
	+ $databases[] = $api->getDatabaseName('directory');
	+ }

	- foreach ($databases as $database) {
	- if ($this->isDryRun()) {
	- $console->writeOut(
	- "%s\n",
	- pht("DRYRUN: Would drop database '%s'.", $database));
	- } else {
	- $console->writeOut(
	- "%s\n",
	- pht("Dropping database '%s'...", $database));
	- queryfx(
	- $api->getConn(null),
	- 'DROP DATABASE IF EXISTS %T',
	- $database);
	- }
	- }
	+ asort($databases);

	- if (!$this->isDryRun()) {
	- $console->writeOut(
	- "%s\n",
	+ foreach ($databases as $database) {
	+ if ($this->isDryRun()) {
	+ $this->logInfo(
	+ pht('DRY RUN'),
	+ pht(
	+ 'Would drop database "%s" on host "%s".',
	+ $database,
	+ $host_display));
	+ } else {
	+ $this->logWarn(
	+ pht('DESTROY'),
	pht(
	- 'Storage on "%s" was destroyed.',
	- $api->getRef()->getRefKey()));
	+ 'Dropping database "%s" on host "%s"...',
	+ $database,
	+ $host_display));
	+
	+ queryfx(
	+ $api->getConn(null),
	+ 'DROP DATABASE IF EXISTS %T',
	+ $database);
	}
	}

	+ if (!$this->isDryRun()) {
	+ $this->logOkay(
	+ pht('DONE'),
	+ pht(
	+ 'Storage on "%s" was destroyed.',
	+ $host_display));
	+ }
	+
	return 0;
	}

	}

File Metadata

Mime Type: text/x-diff
Expires: Mon, Apr 28, 1:58 AM (7 h, 46 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 107640
Default Alt Text: (37 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions