Page MenuHomestyx hydra

No OneTemporary

diff --git a/src/applications/config/check/PhabricatorDatabaseSetupCheck.php b/src/applications/config/check/PhabricatorDatabaseSetupCheck.php
index b5a188c142..7dbc00c9b2 100644
--- a/src/applications/config/check/PhabricatorDatabaseSetupCheck.php
+++ b/src/applications/config/check/PhabricatorDatabaseSetupCheck.php
@@ -1,168 +1,210 @@
<?php
final class PhabricatorDatabaseSetupCheck extends PhabricatorSetupCheck {
public function getDefaultGroup() {
return self::GROUP_IMPORTANT;
}
public function getExecutionOrder() {
// This must run after basic PHP checks, but before most other checks.
return 500;
}
protected function executeChecks() {
$host = PhabricatorEnv::getEnvConfig('mysql.host');
$matches = null;
if (preg_match('/^([^:]+):(\d+)$/', $host, $matches)) {
$host = $matches[1];
$port = $matches[2];
$this->newIssue('storage.mysql.hostport')
->setName(pht('Deprecated mysql.host Format'))
->setSummary(
pht(
'Move port information from `%s` to `%s` in your config.',
'mysql.host',
'mysql.port'))
->setMessage(
pht(
'Your `%s` configuration contains a port number, but this usage '.
'is deprecated. Instead, put the port number in `%s`.',
'mysql.host',
'mysql.port'))
->addPhabricatorConfig('mysql.host')
->addPhabricatorConfig('mysql.port')
->addCommand(
hsprintf(
'<tt>phabricator/ $</tt> ./bin/config set mysql.host %s',
$host))
->addCommand(
hsprintf(
'<tt>phabricator/ $</tt> ./bin/config set mysql.port %s',
$port));
}
- $refs = PhabricatorDatabaseRef::getActiveDatabaseRefs();
+ $refs = PhabricatorDatabaseRef::queryAll();
$refs = mpull($refs, null, 'getRefKey');
// Test if we can connect to each database first. If we can not connect
// to a particular database, we only raise a warning: this allows new web
// nodes to start during a disaster, when some databases may be correctly
// configured but not reachable.
$connect_map = array();
$any_connection = false;
foreach ($refs as $ref_key => $ref) {
$conn_raw = $ref->newManagementConnection();
try {
queryfx($conn_raw, 'SELECT 1');
$database_exception = null;
$any_connection = true;
} catch (AphrontInvalidCredentialsQueryException $ex) {
$database_exception = $ex;
} catch (AphrontConnectionQueryException $ex) {
$database_exception = $ex;
}
if ($database_exception) {
$connect_map[$ref_key] = $database_exception;
unset($refs[$ref_key]);
}
}
if ($connect_map) {
// This is only a fatal error if we could not connect to anything. If
// possible, we still want to start if some database hosts can not be
// reached.
$is_fatal = !$any_connection;
foreach ($connect_map as $ref_key => $database_exception) {
$issue = PhabricatorSetupIssue::newDatabaseConnectionIssue(
$database_exception,
$is_fatal);
$this->addIssue($issue);
}
}
foreach ($refs as $ref_key => $ref) {
if ($this->executeRefChecks($ref)) {
return;
}
}
}
private function executeRefChecks(PhabricatorDatabaseRef $ref) {
$conn_raw = $ref->newManagementConnection();
$ref_key = $ref->getRefKey();
$engines = queryfx_all($conn_raw, 'SHOW ENGINES');
$engines = ipull($engines, 'Support', 'Engine');
$innodb = idx($engines, 'InnoDB');
if ($innodb != 'YES' && $innodb != 'DEFAULT') {
$message = pht(
'The "InnoDB" engine is not available in MySQL (on host "%s"). '.
'Enable InnoDB in your MySQL configuration.'.
"\n\n".
'(If you aleady created tables, MySQL incorrectly used some other '.
'engine to create them. You need to convert them or drop and '.
'reinitialize them.)',
$ref_key);
$this->newIssue('mysql.innodb')
->setName(pht('MySQL InnoDB Engine Not Available'))
->setMessage($message)
->setIsFatal(true);
return true;
}
$namespace = PhabricatorEnv::getEnvConfig('storage.default-namespace');
$databases = queryfx_all($conn_raw, 'SHOW DATABASES');
$databases = ipull($databases, 'Database', 'Database');
if (empty($databases[$namespace.'_meta_data'])) {
$message = pht(
'Run the storage upgrade script to setup databases (host "%s" has '.
'not been initialized).',
$ref_key);
$this->newIssue('storage.upgrade')
->setName(pht('Setup MySQL Schema'))
->setMessage($message)
->setIsFatal(true)
->addCommand(hsprintf('<tt>phabricator/ $</tt> ./bin/storage upgrade'));
return true;
}
$conn_meta = $ref->newApplicationConnection(
$namespace.'_meta_data');
$applied = queryfx_all($conn_meta, 'SELECT patch FROM patch_status');
$applied = ipull($applied, 'patch', 'patch');
$all = PhabricatorSQLPatchList::buildAllPatches();
$diff = array_diff_key($all, $applied);
if ($diff) {
$message = pht(
'Run the storage upgrade script to upgrade databases (host "%s" is '.
'out of date). Missing patches: %s.',
$ref_key,
implode(', ', array_keys($diff)));
$this->newIssue('storage.patch')
->setName(pht('Upgrade MySQL Schema'))
->setIsFatal(true)
->setMessage($message)
->addCommand(
hsprintf('<tt>phabricator/ $</tt> ./bin/storage upgrade'));
return true;
}
+
+ // NOTE: It's possible that replication is broken but we have not been
+ // granted permission to "SHOW SLAVE STATUS" so we can't figure it out.
+ // We allow this kind of configuration and survive these checks, trusting
+ // that operations knows what they're doing. This issue is shown on the
+ // "Database Servers" console.
+
+ switch ($ref->getReplicaStatus()) {
+ case PhabricatorDatabaseRef::REPLICATION_MASTER_REPLICA:
+ $message = pht(
+ 'Database host "%s" is configured as a master, but is replicating '.
+ 'another host. This is dangerous and can mangle or destroy data. '.
+ 'Only replicas should be replicating. Stop replication on the '.
+ 'host or reconfigure Phabricator.',
+ $ref->getRefKey());
+
+ $this->newIssue('db.master.replicating')
+ ->setName(pht('Replicating Master'))
+ ->setIsFatal(true)
+ ->setMessage($message);
+
+ return true;
+ case PhabricatorDatabaseRef::REPLICATION_REPLICA_NONE:
+ case PhabricatorDatabaseRef::REPLICATION_NOT_REPLICATING:
+ if (!$ref->getIsMaster()) {
+ $message = pht(
+ 'Database replica "%s" is listed as a replica, but is not '.
+ 'currently replicating. You are vulnerable to data loss if '.
+ 'the master fails.',
+ $ref->getRefKey());
+
+ // This isn't a fatal because it can normally only put data at risk,
+ // not actually do anything destructive or unrecoverable.
+
+ $this->newIssue('db.replica.not-replicating')
+ ->setName(pht('Nonreplicating Replica'))
+ ->setMessage($message);
+ }
+ break;
+ }
+
+
}
}
diff --git a/src/infrastructure/cluster/PhabricatorDatabaseRef.php b/src/infrastructure/cluster/PhabricatorDatabaseRef.php
index 6b1b7f3612..22f405ad91 100644
--- a/src/infrastructure/cluster/PhabricatorDatabaseRef.php
+++ b/src/infrastructure/cluster/PhabricatorDatabaseRef.php
@@ -1,661 +1,660 @@
<?php
final class PhabricatorDatabaseRef
extends Phobject {
const STATUS_OKAY = 'okay';
const STATUS_FAIL = 'fail';
const STATUS_AUTH = 'auth';
const STATUS_REPLICATION_CLIENT = 'replication-client';
const REPLICATION_OKAY = 'okay';
const REPLICATION_MASTER_REPLICA = 'master-replica';
const REPLICATION_REPLICA_NONE = 'replica-none';
const REPLICATION_SLOW = 'replica-slow';
const REPLICATION_NOT_REPLICATING = 'not-replicating';
const KEY_REFS = 'cluster.db.refs';
const KEY_INDIVIDUAL = 'cluster.db.individual';
private $host;
private $port;
private $user;
private $pass;
private $disabled;
private $isMaster;
private $isIndividual;
private $connectionLatency;
private $connectionStatus;
private $connectionMessage;
private $replicaStatus;
private $replicaMessage;
private $replicaDelay;
private $healthRecord;
private $didFailToConnect;
private $isDefaultPartition;
private $applicationMap = array();
private $masterRef;
private $replicaRefs = array();
public function setHost($host) {
$this->host = $host;
return $this;
}
public function getHost() {
return $this->host;
}
public function setPort($port) {
$this->port = $port;
return $this;
}
public function getPort() {
return $this->port;
}
public function setUser($user) {
$this->user = $user;
return $this;
}
public function getUser() {
return $this->user;
}
public function setPass(PhutilOpaqueEnvelope $pass) {
$this->pass = $pass;
return $this;
}
public function getPass() {
return $this->pass;
}
public function setIsMaster($is_master) {
$this->isMaster = $is_master;
return $this;
}
public function getIsMaster() {
return $this->isMaster;
}
public function setDisabled($disabled) {
$this->disabled = $disabled;
return $this;
}
public function getDisabled() {
return $this->disabled;
}
public function setConnectionLatency($connection_latency) {
$this->connectionLatency = $connection_latency;
return $this;
}
public function getConnectionLatency() {
return $this->connectionLatency;
}
public function setConnectionStatus($connection_status) {
$this->connectionStatus = $connection_status;
return $this;
}
public function getConnectionStatus() {
if ($this->connectionStatus === null) {
throw new PhutilInvalidStateException('queryAll');
}
return $this->connectionStatus;
}
public function setConnectionMessage($connection_message) {
$this->connectionMessage = $connection_message;
return $this;
}
public function getConnectionMessage() {
return $this->connectionMessage;
}
public function setReplicaStatus($replica_status) {
$this->replicaStatus = $replica_status;
return $this;
}
public function getReplicaStatus() {
return $this->replicaStatus;
}
public function setReplicaMessage($replica_message) {
$this->replicaMessage = $replica_message;
return $this;
}
public function getReplicaMessage() {
return $this->replicaMessage;
}
public function setReplicaDelay($replica_delay) {
$this->replicaDelay = $replica_delay;
return $this;
}
public function getReplicaDelay() {
return $this->replicaDelay;
}
public function setIsIndividual($is_individual) {
$this->isIndividual = $is_individual;
return $this;
}
public function getIsIndividual() {
return $this->isIndividual;
}
public function setIsDefaultPartition($is_default_partition) {
$this->isDefaultPartition = $is_default_partition;
return $this;
}
public function getIsDefaultPartition() {
return $this->isDefaultPartition;
}
public function setApplicationMap(array $application_map) {
$this->applicationMap = $application_map;
return $this;
}
public function getApplicationMap() {
return $this->applicationMap;
}
public function setMasterRef(PhabricatorDatabaseRef $master_ref) {
$this->masterRef = $master_ref;
return $this;
}
public function getMasterRef() {
return $this->masterRef;
}
public function addReplicaRef(PhabricatorDatabaseRef $replica_ref) {
$this->replicaRefs[] = $replica_ref;
return $this;
}
public function getReplicaRefs() {
return $this->replicaRefs;
}
public function getRefKey() {
$host = $this->getHost();
$port = $this->getPort();
if (strlen($port)) {
return "{$host}:{$port}";
}
return $host;
}
public static function getConnectionStatusMap() {
return array(
self::STATUS_OKAY => array(
'icon' => 'fa-exchange',
'color' => 'green',
'label' => pht('Okay'),
),
self::STATUS_FAIL => array(
'icon' => 'fa-times',
'color' => 'red',
'label' => pht('Failed'),
),
self::STATUS_AUTH => array(
'icon' => 'fa-key',
'color' => 'red',
'label' => pht('Invalid Credentials'),
),
self::STATUS_REPLICATION_CLIENT => array(
'icon' => 'fa-eye-slash',
'color' => 'yellow',
'label' => pht('Missing Permission'),
),
);
}
public static function getReplicaStatusMap() {
return array(
self::REPLICATION_OKAY => array(
'icon' => 'fa-download',
'color' => 'green',
'label' => pht('Okay'),
),
self::REPLICATION_MASTER_REPLICA => array(
'icon' => 'fa-database',
'color' => 'red',
'label' => pht('Replicating Master'),
),
self::REPLICATION_REPLICA_NONE => array(
'icon' => 'fa-download',
'color' => 'red',
'label' => pht('Not A Replica'),
),
self::REPLICATION_SLOW => array(
'icon' => 'fa-hourglass',
'color' => 'red',
'label' => pht('Slow Replication'),
),
self::REPLICATION_NOT_REPLICATING => array(
'icon' => 'fa-exclamation-triangle',
'color' => 'red',
'label' => pht('Not Replicating'),
),
);
}
public static function getClusterRefs() {
$cache = PhabricatorCaches::getRequestCache();
$refs = $cache->getKey(self::KEY_REFS);
if (!$refs) {
$refs = self::newRefs();
$cache->setKey(self::KEY_REFS, $refs);
}
return $refs;
}
public static function getLiveIndividualRef() {
$cache = PhabricatorCaches::getRequestCache();
$ref = $cache->getKey(self::KEY_INDIVIDUAL);
if (!$ref) {
$ref = self::newIndividualRef();
$cache->setKey(self::KEY_INDIVIDUAL, $ref);
}
return $ref;
}
public static function newRefs() {
$default_port = PhabricatorEnv::getEnvConfig('mysql.port');
$default_port = nonempty($default_port, 3306);
$default_user = PhabricatorEnv::getEnvConfig('mysql.user');
$default_pass = PhabricatorEnv::getEnvConfig('mysql.pass');
$default_pass = new PhutilOpaqueEnvelope($default_pass);
$config = PhabricatorEnv::getEnvConfig('cluster.databases');
return id(new PhabricatorDatabaseRefParser())
->setDefaultPort($default_port)
->setDefaultUser($default_user)
->setDefaultPass($default_pass)
->newRefs($config);
}
public static function queryAll() {
- $refs = self::newRefs();
+ $refs = self::getActiveDatabaseRefs();
+ return self::queryRefs($refs);
+ }
+ private static function queryRefs(array $refs) {
foreach ($refs as $ref) {
- if ($ref->getDisabled()) {
- continue;
- }
-
$conn = $ref->newManagementConnection();
$t_start = microtime(true);
$replica_status = false;
try {
$replica_status = queryfx_one($conn, 'SHOW SLAVE STATUS');
$ref->setConnectionStatus(self::STATUS_OKAY);
} catch (AphrontAccessDeniedQueryException $ex) {
$ref->setConnectionStatus(self::STATUS_REPLICATION_CLIENT);
$ref->setConnectionMessage(
pht(
'No permission to run "SHOW SLAVE STATUS". Grant this user '.
'"REPLICATION CLIENT" permission to allow Phabricator to '.
'monitor replica health.'));
} catch (AphrontInvalidCredentialsQueryException $ex) {
$ref->setConnectionStatus(self::STATUS_AUTH);
$ref->setConnectionMessage($ex->getMessage());
} catch (AphrontQueryException $ex) {
$ref->setConnectionStatus(self::STATUS_FAIL);
$class = get_class($ex);
$message = $ex->getMessage();
$ref->setConnectionMessage(
pht(
'%s: %s',
get_class($ex),
$ex->getMessage()));
}
$t_end = microtime(true);
$ref->setConnectionLatency($t_end - $t_start);
if ($replica_status !== false) {
$is_replica = (bool)$replica_status;
if ($ref->getIsMaster() && $is_replica) {
$ref->setReplicaStatus(self::REPLICATION_MASTER_REPLICA);
$ref->setReplicaMessage(
pht(
'This host has a "master" role, but is replicating data from '.
'another host ("%s")!',
idx($replica_status, 'Master_Host')));
} else if (!$ref->getIsMaster() && !$is_replica) {
$ref->setReplicaStatus(self::REPLICATION_REPLICA_NONE);
$ref->setReplicaMessage(
pht(
'This host has a "replica" role, but is not replicating data '.
'from a master (no output from "SHOW SLAVE STATUS").'));
} else {
$ref->setReplicaStatus(self::REPLICATION_OKAY);
}
if ($is_replica) {
$latency = idx($replica_status, 'Seconds_Behind_Master');
if (!strlen($latency)) {
$ref->setReplicaStatus(self::REPLICATION_NOT_REPLICATING);
} else {
$latency = (int)$latency;
$ref->setReplicaDelay($latency);
if ($latency > 30) {
$ref->setReplicaStatus(self::REPLICATION_SLOW);
$ref->setReplicaMessage(
pht(
'This replica is lagging far behind the master. Data is at '.
'risk!'));
}
}
}
}
}
return $refs;
}
public function newManagementConnection() {
return $this->newConnection(
array(
'retries' => 0,
'timeout' => 2,
));
}
public function newApplicationConnection($database) {
return $this->newConnection(
array(
'database' => $database,
));
}
public function isSevered() {
// If we only have an individual database, never sever our connection to
// it, at least for now. It's possible that using the same severing rules
// might eventually make sense to help alleviate load-related failures,
// but we should wait for all the cluster stuff to stabilize first.
if ($this->getIsIndividual()) {
return false;
}
if ($this->didFailToConnect) {
return true;
}
$record = $this->getHealthRecord();
$is_healthy = $record->getIsHealthy();
if (!$is_healthy) {
return true;
}
return false;
}
public function isReachable(AphrontDatabaseConnection $connection) {
$record = $this->getHealthRecord();
$should_check = $record->getShouldCheck();
if ($this->isSevered() && !$should_check) {
return false;
}
try {
$connection->openConnection();
$reachable = true;
} catch (AphrontSchemaQueryException $ex) {
// We get one of these if the database we're trying to select does not
// exist. In this case, just re-throw the exception. This is expected
// during first-time setup, when databases like "config" will not exist
// yet.
throw $ex;
} catch (Exception $ex) {
$reachable = false;
}
if ($should_check) {
$record->didHealthCheck($reachable);
}
if (!$reachable) {
$this->didFailToConnect = true;
}
return $reachable;
}
public function checkHealth() {
$health = $this->getHealthRecord();
$should_check = $health->getShouldCheck();
if ($should_check) {
// This does an implicit health update.
$connection = $this->newManagementConnection();
$this->isReachable($connection);
}
return $this;
}
public function getHealthRecord() {
if (!$this->healthRecord) {
$this->healthRecord = new PhabricatorDatabaseHealthRecord($this);
}
return $this->healthRecord;
}
public static function getActiveDatabaseRefs() {
$refs = array();
foreach (self::getMasterDatabaseRefs() as $ref) {
$refs[] = $ref;
}
foreach (self::getReplicaDatabaseRefs() as $ref) {
$refs[] = $ref;
}
return $refs;
}
public static function getAllMasterDatabaseRefs() {
$refs = self::getClusterRefs();
if (!$refs) {
return array(self::getLiveIndividualRef());
}
$masters = array();
foreach ($refs as $ref) {
if ($ref->getDisabled()) {
continue;
}
if ($ref->getIsMaster()) {
$masters[] = $ref;
}
}
return $masters;
}
public static function getMasterDatabaseRefs() {
$refs = self::getAllMasterDatabaseRefs();
return self::getEnabledRefs($refs);
}
public function isApplicationHost($database) {
return isset($this->applicationMap[$database]);
}
public static function getMasterDatabaseRefForApplication($application) {
$masters = self::getMasterDatabaseRefs();
$application_master = null;
$default_master = null;
foreach ($masters as $master) {
if ($master->isApplicationHost($application)) {
$application_master = $master;
break;
}
if ($master->getIsDefaultPartition()) {
$default_master = $master;
}
}
if ($application_master) {
$masters = array($application_master);
} else if ($default_master) {
$masters = array($default_master);
} else {
$masters = array();
}
$masters = self::getEnabledRefs($masters);
$master = head($masters);
return $master;
}
public static function newIndividualRef() {
$conf = PhabricatorEnv::newObjectFromConfig(
'mysql.configuration-provider',
array(null, 'w', null));
return id(new self())
->setHost($conf->getHost())
->setPort($conf->getPort())
->setUser($conf->getUser())
->setPass($conf->getPassword())
->setIsIndividual(true)
->setIsMaster(true);
}
public static function getAllReplicaDatabaseRefs() {
$refs = self::getClusterRefs();
if (!$refs) {
return array();
}
$replicas = array();
foreach ($refs as $ref) {
if ($ref->getIsMaster()) {
continue;
}
$replicas[] = $ref;
}
return $replicas;
}
public static function getReplicaDatabaseRefs() {
$refs = self::getAllReplicaDatabaseRefs();
return self::getEnabledRefs($refs);
}
private static function getEnabledRefs(array $refs) {
foreach ($refs as $key => $ref) {
if ($ref->getDisabled()) {
unset($refs[$key]);
}
}
return $refs;
}
public static function getReplicaDatabaseRefForApplication($application) {
$replicas = self::getReplicaDatabaseRefs();
$application_replicas = array();
$default_replicas = array();
foreach ($replicas as $replica) {
$master = $replica->getMaster();
if ($master->isApplicationHost($application)) {
$application_replicas[] = $replica;
}
if ($master->getIsDefaultPartition()) {
$default_replicas[] = $replica;
}
}
if ($application_replicas) {
$replicas = $application_replicas;
} else {
$replicas = $default_replicas;
}
$replicas = self::getEnabledRefs($replicas);
// TODO: We may have multiple replicas to choose from, and could make
// more of an effort to pick the "best" one here instead of always
// picking the first one. Once we've picked one, we should try to use
// the same replica for the rest of the request, though.
return head($replicas);
}
private function newConnection(array $options) {
// If we believe the database is unhealthy, don't spend as much time
// trying to connect to it, since it's likely to continue to fail and
// hammering it can only make the problem worse.
$record = $this->getHealthRecord();
if ($record->getIsHealthy()) {
$default_retries = 3;
$default_timeout = 10;
} else {
$default_retries = 0;
$default_timeout = 2;
}
$spec = $options + array(
'user' => $this->getUser(),
'pass' => $this->getPass(),
'host' => $this->getHost(),
'port' => $this->getPort(),
'database' => null,
'retries' => $default_retries,
'timeout' => $default_timeout,
);
return PhabricatorEnv::newObjectFromConfig(
'mysql.implementation',
array(
$spec,
));
}
}
diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php
index 2404918b56..5fb117eb51 100644
--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php
+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php
@@ -1,1019 +1,1021 @@
<?php
abstract class PhabricatorStorageManagementWorkflow
extends PhabricatorManagementWorkflow {
private $apis = array();
private $dryRun;
private $force;
private $patches;
private $didInitialize;
final public function setAPIs(array $apis) {
$this->apis = $apis;
return $this;
}
final public function getAnyAPI() {
return head($this->getAPIs());
}
final public function getMasterAPIs() {
$apis = $this->getAPIs();
$results = array();
foreach ($apis as $api) {
if ($api->getRef()->getIsMaster()) {
$results[] = $api;
}
}
if (!$results) {
throw new PhutilArgumentUsageException(
pht(
'This command only operates on database masters, but the selected '.
'database hosts do not include any masters.'));
}
return $results;
}
final public function getSingleAPI() {
$apis = $this->getAPIs();
if (count($apis) == 1) {
return head($apis);
}
throw new PhutilArgumentUsageException(
pht(
'Phabricator is configured in cluster mode, with multiple database '.
'hosts. Use "--host" to specify which host you want to operate on.'));
}
final public function getAPIs() {
return $this->apis;
}
final protected function isDryRun() {
return $this->dryRun;
}
final protected function setDryRun($dry_run) {
$this->dryRun = $dry_run;
return $this;
}
final protected function isForce() {
return $this->force;
}
final protected function setForce($force) {
$this->force = $force;
return $this;
}
public function getPatches() {
return $this->patches;
}
public function setPatches(array $patches) {
assert_instances_of($patches, 'PhabricatorStoragePatch');
$this->patches = $patches;
return $this;
}
protected function isReadOnlyWorkflow() {
return false;
}
public function execute(PhutilArgumentParser $args) {
$this->setDryRun($args->getArg('dryrun'));
$this->setForce($args->getArg('force'));
if (!$this->isReadOnlyWorkflow()) {
if (PhabricatorEnv::isReadOnly()) {
if ($this->isForce()) {
PhabricatorEnv::setReadOnly(false, null);
} else {
throw new PhutilArgumentUsageException(
pht(
'Phabricator is currently in read-only mode. Use --force to '.
'override this mode.'));
}
}
}
return $this->didExecute($args);
}
public function didExecute(PhutilArgumentParser $args) {}
private function loadSchemata(PhabricatorStorageManagementAPI $api) {
$query = id(new PhabricatorConfigSchemaQuery());
$ref = $api->getRef();
$ref_key = $ref->getRefKey();
$query->setAPIs(array($api));
$query->setRefs(array($ref));
$actual = $query->loadActualSchemata();
$expect = $query->loadExpectedSchemata();
$comp = $query->buildComparisonSchemata($expect, $actual);
return array(
$comp[$ref_key],
$expect[$ref_key],
$actual[$ref_key],
);
}
final protected function adjustSchemata(
PhabricatorStorageManagementAPI $api,
$unsafe) {
$lock = $this->lock($api);
try {
$err = $this->doAdjustSchemata($api, $unsafe);
} catch (Exception $ex) {
$lock->unlock();
throw $ex;
}
$lock->unlock();
return $err;
}
final private function doAdjustSchemata(
PhabricatorStorageManagementAPI $api,
$unsafe) {
$console = PhutilConsole::getConsole();
$console->writeOut(
"%s\n",
pht(
'Verifying database schemata on "%s"...',
$api->getRef()->getRefKey()));
list($adjustments, $errors) = $this->findAdjustments($api);
if (!$adjustments) {
$console->writeOut(
"%s\n",
pht('Found no adjustments for schemata.'));
return $this->printErrors($errors, 0);
}
if (!$this->force && !$api->isCharacterSetAvailable('utf8mb4')) {
$message = pht(
"You have an old version of MySQL (older than 5.5) which does not ".
"support the utf8mb4 character set. We strongly recomend upgrading to ".
"5.5 or newer.\n\n".
"If you apply adjustments now and later update MySQL to 5.5 or newer, ".
"you'll need to apply adjustments again (and they will take a long ".
"time).\n\n".
"You can exit this workflow, update MySQL now, and then run this ".
"workflow again. This is recommended, but may cause a lot of downtime ".
"right now.\n\n".
"You can exit this workflow, continue using Phabricator without ".
"applying adjustments, update MySQL at a later date, and then run ".
"this workflow again. This is also a good approach, and will let you ".
"delay downtime until later.\n\n".
"You can proceed with this workflow, and then optionally update ".
"MySQL at a later date. After you do, you'll need to apply ".
"adjustments again.\n\n".
"For more information, see \"Managing Storage Adjustments\" in ".
"the documentation.");
$console->writeOut(
"\n**<bg:yellow> %s </bg>**\n\n%s\n",
pht('OLD MySQL VERSION'),
phutil_console_wrap($message));
$prompt = pht('Continue with old MySQL version?');
if (!phutil_console_confirm($prompt, $default_no = true)) {
return;
}
}
$table = id(new PhutilConsoleTable())
->addColumn('database', array('title' => pht('Database')))
->addColumn('table', array('title' => pht('Table')))
->addColumn('name', array('title' => pht('Name')))
->addColumn('info', array('title' => pht('Issues')));
foreach ($adjustments as $adjust) {
$info = array();
foreach ($adjust['issues'] as $issue) {
$info[] = PhabricatorConfigStorageSchema::getIssueName($issue);
}
$table->addRow(array(
'database' => $adjust['database'],
'table' => idx($adjust, 'table'),
'name' => idx($adjust, 'name'),
'info' => implode(', ', $info),
));
}
$console->writeOut("\n\n");
$table->draw();
if ($this->dryRun) {
$console->writeOut(
"%s\n",
pht('DRYRUN: Would apply adjustments.'));
return 0;
} else if ($this->didInitialize) {
// If we just initialized the database, continue without prompting. This
// is nicer for first-time setup and there's no reasonable reason any
// user would ever answer "no" to the prompt against an empty schema.
} else if (!$this->force) {
$console->writeOut(
"\n%s\n",
pht(
"Found %s adjustment(s) to apply, detailed above.\n\n".
"You can review adjustments in more detail from the web interface, ".
"in Config > Database Status. To better understand the adjustment ".
"workflow, see \"Managing Storage Adjustments\" in the ".
"documentation.\n\n".
"MySQL needs to copy table data to make some adjustments, so these ".
"migrations may take some time.",
phutil_count($adjustments)));
$prompt = pht('Apply these schema adjustments?');
if (!phutil_console_confirm($prompt, $default_no = true)) {
return 1;
}
}
$console->writeOut(
"%s\n",
pht('Applying schema adjustments...'));
$conn = $api->getConn(null);
if ($unsafe) {
queryfx($conn, 'SET SESSION sql_mode = %s', '');
} else {
queryfx($conn, 'SET SESSION sql_mode = %s', 'STRICT_ALL_TABLES');
}
$failed = array();
// We make changes in several phases.
$phases = array(
// Drop surplus autoincrements. This allows us to drop primary keys on
// autoincrement columns.
'drop_auto',
// Drop all keys we're going to adjust. This prevents them from
// interfering with column changes.
'drop_keys',
// Apply all database, table, and column changes.
'main',
// Restore adjusted keys.
'add_keys',
// Add missing autoincrements.
'add_auto',
);
$bar = id(new PhutilConsoleProgressBar())
->setTotal(count($adjustments) * count($phases));
foreach ($phases as $phase) {
foreach ($adjustments as $adjust) {
try {
switch ($adjust['kind']) {
case 'database':
if ($phase == 'main') {
queryfx(
$conn,
'ALTER DATABASE %T CHARACTER SET = %s COLLATE = %s',
$adjust['database'],
$adjust['charset'],
$adjust['collation']);
}
break;
case 'table':
if ($phase == 'main') {
queryfx(
$conn,
'ALTER TABLE %T.%T COLLATE = %s',
$adjust['database'],
$adjust['table'],
$adjust['collation']);
}
break;
case 'column':
$apply = false;
$auto = false;
$new_auto = idx($adjust, 'auto');
if ($phase == 'drop_auto') {
if ($new_auto === false) {
$apply = true;
$auto = false;
}
} else if ($phase == 'main') {
$apply = true;
if ($new_auto === false) {
$auto = false;
} else {
$auto = $adjust['is_auto'];
}
} else if ($phase == 'add_auto') {
if ($new_auto === true) {
$apply = true;
$auto = true;
}
}
if ($apply) {
$parts = array();
if ($auto) {
$parts[] = qsprintf(
$conn,
'AUTO_INCREMENT');
}
if ($adjust['charset']) {
$parts[] = qsprintf(
$conn,
'CHARACTER SET %Q COLLATE %Q',
$adjust['charset'],
$adjust['collation']);
}
queryfx(
$conn,
'ALTER TABLE %T.%T MODIFY %T %Q %Q %Q',
$adjust['database'],
$adjust['table'],
$adjust['name'],
$adjust['type'],
implode(' ', $parts),
$adjust['nullable'] ? 'NULL' : 'NOT NULL');
}
break;
case 'key':
if (($phase == 'drop_keys') && $adjust['exists']) {
if ($adjust['name'] == 'PRIMARY') {
$key_name = 'PRIMARY KEY';
} else {
$key_name = qsprintf($conn, 'KEY %T', $adjust['name']);
}
queryfx(
$conn,
'ALTER TABLE %T.%T DROP %Q',
$adjust['database'],
$adjust['table'],
$key_name);
}
if (($phase == 'add_keys') && $adjust['keep']) {
// Different keys need different creation syntax. Notable
// special cases are primary keys and fulltext keys.
if ($adjust['name'] == 'PRIMARY') {
$key_name = 'PRIMARY KEY';
} else if ($adjust['indexType'] == 'FULLTEXT') {
$key_name = qsprintf($conn, 'FULLTEXT %T', $adjust['name']);
} else {
if ($adjust['unique']) {
$key_name = qsprintf(
$conn,
'UNIQUE KEY %T',
$adjust['name']);
} else {
$key_name = qsprintf(
$conn,
'/* NONUNIQUE */ KEY %T',
$adjust['name']);
}
}
queryfx(
$conn,
'ALTER TABLE %T.%T ADD %Q (%Q)',
$adjust['database'],
$adjust['table'],
$key_name,
implode(', ', $adjust['columns']));
}
break;
default:
throw new Exception(
pht('Unknown schema adjustment kind "%s"!', $adjust['kind']));
}
} catch (AphrontQueryException $ex) {
$failed[] = array($adjust, $ex);
}
$bar->update(1);
}
}
$bar->done();
if (!$failed) {
$console->writeOut(
"%s\n",
pht('Completed applying all schema adjustments.'));
$err = 0;
} else {
$table = id(new PhutilConsoleTable())
->addColumn('target', array('title' => pht('Target')))
->addColumn('error', array('title' => pht('Error')));
foreach ($failed as $failure) {
list($adjust, $ex) = $failure;
$pieces = array_select_keys(
$adjust,
array('database', 'table', 'name'));
$pieces = array_filter($pieces);
$target = implode('.', $pieces);
$table->addRow(
array(
'target' => $target,
'error' => $ex->getMessage(),
));
}
$console->writeOut("\n");
$table->draw();
$console->writeOut(
"\n%s\n",
pht('Failed to make some schema adjustments, detailed above.'));
$console->writeOut(
"%s\n",
pht(
'For help troubleshooting adjustments, see "Managing Storage '.
'Adjustments" in the documentation.'));
$err = 1;
}
return $this->printErrors($errors, $err);
}
private function findAdjustments(
PhabricatorStorageManagementAPI $api) {
list($comp, $expect, $actual) = $this->loadSchemata($api);
$issue_charset = PhabricatorConfigStorageSchema::ISSUE_CHARSET;
$issue_collation = PhabricatorConfigStorageSchema::ISSUE_COLLATION;
$issue_columntype = PhabricatorConfigStorageSchema::ISSUE_COLUMNTYPE;
$issue_surpluskey = PhabricatorConfigStorageSchema::ISSUE_SURPLUSKEY;
$issue_missingkey = PhabricatorConfigStorageSchema::ISSUE_MISSINGKEY;
$issue_columns = PhabricatorConfigStorageSchema::ISSUE_KEYCOLUMNS;
$issue_unique = PhabricatorConfigStorageSchema::ISSUE_UNIQUE;
$issue_longkey = PhabricatorConfigStorageSchema::ISSUE_LONGKEY;
$issue_auto = PhabricatorConfigStorageSchema::ISSUE_AUTOINCREMENT;
$adjustments = array();
$errors = array();
foreach ($comp->getDatabases() as $database_name => $database) {
foreach ($this->findErrors($database) as $issue) {
$errors[] = array(
'database' => $database_name,
'issue' => $issue,
);
}
$expect_database = $expect->getDatabase($database_name);
$actual_database = $actual->getDatabase($database_name);
if (!$expect_database || !$actual_database) {
// If there's a real issue here, skip this stuff.
continue;
}
if ($actual_database->getAccessDenied()) {
// If we can't access the database, we can't access the tables either.
continue;
}
$issues = array();
if ($database->hasIssue($issue_charset)) {
$issues[] = $issue_charset;
}
if ($database->hasIssue($issue_collation)) {
$issues[] = $issue_collation;
}
if ($issues) {
$adjustments[] = array(
'kind' => 'database',
'database' => $database_name,
'issues' => $issues,
'charset' => $expect_database->getCharacterSet(),
'collation' => $expect_database->getCollation(),
);
}
foreach ($database->getTables() as $table_name => $table) {
foreach ($this->findErrors($table) as $issue) {
$errors[] = array(
'database' => $database_name,
'table' => $table_name,
'issue' => $issue,
);
}
$expect_table = $expect_database->getTable($table_name);
$actual_table = $actual_database->getTable($table_name);
if (!$expect_table || !$actual_table) {
continue;
}
$issues = array();
if ($table->hasIssue($issue_collation)) {
$issues[] = $issue_collation;
}
if ($issues) {
$adjustments[] = array(
'kind' => 'table',
'database' => $database_name,
'table' => $table_name,
'issues' => $issues,
'collation' => $expect_table->getCollation(),
);
}
foreach ($table->getColumns() as $column_name => $column) {
foreach ($this->findErrors($column) as $issue) {
$errors[] = array(
'database' => $database_name,
'table' => $table_name,
'name' => $column_name,
'issue' => $issue,
);
}
$expect_column = $expect_table->getColumn($column_name);
$actual_column = $actual_table->getColumn($column_name);
if (!$expect_column || !$actual_column) {
continue;
}
$issues = array();
if ($column->hasIssue($issue_collation)) {
$issues[] = $issue_collation;
}
if ($column->hasIssue($issue_charset)) {
$issues[] = $issue_charset;
}
if ($column->hasIssue($issue_columntype)) {
$issues[] = $issue_columntype;
}
if ($column->hasIssue($issue_auto)) {
$issues[] = $issue_auto;
}
if ($issues) {
if ($expect_column->getCharacterSet() === null) {
// For non-text columns, we won't be specifying a collation or
// character set.
$charset = null;
$collation = null;
} else {
$charset = $expect_column->getCharacterSet();
$collation = $expect_column->getCollation();
}
$adjustment = array(
'kind' => 'column',
'database' => $database_name,
'table' => $table_name,
'name' => $column_name,
'issues' => $issues,
'collation' => $collation,
'charset' => $charset,
'type' => $expect_column->getColumnType(),
// NOTE: We don't adjust column nullability because it is
// dangerous, so always use the current nullability.
'nullable' => $actual_column->getNullable(),
// NOTE: This always stores the current value, because we have
// to make these updates separately.
'is_auto' => $actual_column->getAutoIncrement(),
);
if ($column->hasIssue($issue_auto)) {
$adjustment['auto'] = $expect_column->getAutoIncrement();
}
$adjustments[] = $adjustment;
}
}
foreach ($table->getKeys() as $key_name => $key) {
foreach ($this->findErrors($key) as $issue) {
$errors[] = array(
'database' => $database_name,
'table' => $table_name,
'name' => $key_name,
'issue' => $issue,
);
}
$expect_key = $expect_table->getKey($key_name);
$actual_key = $actual_table->getKey($key_name);
$issues = array();
$keep_key = true;
if ($key->hasIssue($issue_surpluskey)) {
$issues[] = $issue_surpluskey;
$keep_key = false;
}
if ($key->hasIssue($issue_missingkey)) {
$issues[] = $issue_missingkey;
}
if ($key->hasIssue($issue_columns)) {
$issues[] = $issue_columns;
}
if ($key->hasIssue($issue_unique)) {
$issues[] = $issue_unique;
}
// NOTE: We can't really fix this, per se, but we may need to remove
// the key to change the column type. In the best case, the new
// column type won't be overlong and recreating the key really will
// fix the issue. In the worst case, we get the right column type and
// lose the key, which is still better than retaining the key having
// the wrong column type.
if ($key->hasIssue($issue_longkey)) {
$issues[] = $issue_longkey;
}
if ($issues) {
$adjustment = array(
'kind' => 'key',
'database' => $database_name,
'table' => $table_name,
'name' => $key_name,
'issues' => $issues,
'exists' => (bool)$actual_key,
'keep' => $keep_key,
);
if ($keep_key) {
$adjustment += array(
'columns' => $expect_key->getColumnNames(),
'unique' => $expect_key->getUnique(),
'indexType' => $expect_key->getIndexType(),
);
}
$adjustments[] = $adjustment;
}
}
}
}
return array($adjustments, $errors);
}
private function findErrors(PhabricatorConfigStorageSchema $schema) {
$result = array();
foreach ($schema->getLocalIssues() as $issue) {
$status = PhabricatorConfigStorageSchema::getIssueStatus($issue);
if ($status == PhabricatorConfigStorageSchema::STATUS_FAIL) {
$result[] = $issue;
}
}
return $result;
}
private function printErrors(array $errors, $default_return) {
if (!$errors) {
return $default_return;
}
$console = PhutilConsole::getConsole();
$table = id(new PhutilConsoleTable())
->addColumn('target', array('title' => pht('Target')))
->addColumn('error', array('title' => pht('Error')));
$any_surplus = false;
$all_surplus = true;
$any_access = false;
$all_access = true;
foreach ($errors as $error) {
$pieces = array_select_keys(
$error,
array('database', 'table', 'name'));
$pieces = array_filter($pieces);
$target = implode('.', $pieces);
$name = PhabricatorConfigStorageSchema::getIssueName($error['issue']);
$issue = $error['issue'];
if ($issue === PhabricatorConfigStorageSchema::ISSUE_SURPLUS) {
$any_surplus = true;
} else {
$all_surplus = false;
}
if ($issue === PhabricatorConfigStorageSchema::ISSUE_ACCESSDENIED) {
$any_access = true;
} else {
$all_access = false;
}
$table->addRow(
array(
'target' => $target,
'error' => $name,
));
}
$console->writeOut("\n");
$table->draw();
$console->writeOut("\n");
$message = array();
if ($all_surplus) {
$message[] = pht(
'You have surplus schemata (extra tables or columns which Phabricator '.
'does not expect). For information on resolving these '.
'issues, see the "Surplus Schemata" section in the "Managing Storage '.
'Adjustments" article in the documentation.');
} else if ($all_access) {
$message[] = pht(
'The user you are connecting to MySQL with does not have the correct '.
'permissions, and can not access some databases or tables that it '.
'needs to be able to access. GRANT the user additional permissions.');
} else {
$message[] = pht(
'The schemata have errors (detailed above) which the adjustment '.
'workflow can not fix.');
if ($any_access) {
$message[] = pht(
'Some of these errors are caused by access control problems. '.
'The user you are connecting with does not have permission to see '.
'all of the database or tables that Phabricator uses. You need to '.
'GRANT the user more permission, or use a different user.');
}
if ($any_surplus) {
$message[] = pht(
'Some of these errors are caused by surplus schemata (extra '.
'tables or columns which Phabricator does not expect). These are '.
'not serious. For information on resolving these issues, see the '.
'"Surplus Schemata" section in the "Managing Storage Adjustments" '.
'article in the documentation.');
}
$message[] = pht(
'If you are not developing Phabricator itself, report this issue to '.
'the upstream.');
$message[] = pht(
'If you are developing Phabricator, these errors usually indicate '.
'that your schema specifications do not agree with the schemata your '.
'code actually builds.');
}
$message = implode("\n\n", $message);
if ($all_surplus) {
$console->writeOut(
"**<bg:yellow> %s </bg>**\n\n%s\n",
pht('SURPLUS SCHEMATA'),
phutil_console_wrap($message));
} else if ($all_access) {
$console->writeOut(
"**<bg:yellow> %s </bg>**\n\n%s\n",
pht('ACCESS DENIED'),
phutil_console_wrap($message));
} else {
$console->writeOut(
"**<bg:red> %s </bg>**\n\n%s\n",
pht('SCHEMATA ERRORS'),
phutil_console_wrap($message));
}
return 2;
}
final protected function upgradeSchemata(
PhabricatorStorageManagementAPI $api,
$apply_only = null,
$no_quickstart = false,
$init_only = false) {
$lock = $this->lock($api);
try {
$this->doUpgradeSchemata($api, $apply_only, $no_quickstart, $init_only);
} catch (Exception $ex) {
$lock->unlock();
throw $ex;
}
$lock->unlock();
}
final private function doUpgradeSchemata(
PhabricatorStorageManagementAPI $api,
$apply_only,
$no_quickstart,
$init_only) {
+ $patches = $this->patches;
+
$applied = $api->getAppliedPatches();
if ($applied === null) {
if ($this->dryRun) {
echo pht(
"DRYRUN: Patch metadata storage doesn't exist yet, ".
"it would be created.\n");
return 0;
}
if ($apply_only) {
throw new PhutilArgumentUsageException(
pht(
'Storage has not been initialized yet, you must initialize '.
'storage before selectively applying patches.'));
return 1;
}
// If we're initializing storage for the first time, track it so that
// we can give the user a nicer experience during the subsequent
// adjustment phase.
$this->didInitialize = true;
- $legacy = $api->getLegacyPatches($this->patches);
+ $legacy = $api->getLegacyPatches($patches);
if ($legacy || $no_quickstart || $init_only) {
// If we have legacy patches, we can't quickstart.
$api->createDatabase('meta_data');
$api->createTable(
'meta_data',
'patch_status',
array(
'patch VARCHAR(255) NOT NULL PRIMARY KEY COLLATE utf8_general_ci',
'applied INT UNSIGNED NOT NULL',
));
foreach ($legacy as $patch) {
$api->markPatchApplied($patch);
}
} else {
echo pht('Loading quickstart template...')."\n";
$root = dirname(phutil_get_library_root('phabricator'));
$sql = $root.'/resources/sql/quickstart.sql';
$api->applyPatchSQL($sql);
}
}
if ($init_only) {
echo pht('Storage initialized.')."\n";
return 0;
}
$applied = $api->getAppliedPatches();
$applied = array_fuse($applied);
$skip_mark = false;
if ($apply_only) {
if (isset($applied[$apply_only])) {
unset($applied[$apply_only]);
$skip_mark = true;
if (!$this->force && !$this->dryRun) {
echo phutil_console_wrap(
pht(
"Patch '%s' has already been applied. Are you sure you want ".
"to apply it again? This may put your storage in a state ".
"that the upgrade scripts can not automatically manage.",
$apply_only));
if (!phutil_console_confirm(pht('Apply patch again?'))) {
echo pht('Cancelled.')."\n";
return 1;
}
}
}
}
while (true) {
$applied_something = false;
- foreach ($this->patches as $key => $patch) {
+ foreach ($patches as $key => $patch) {
if (isset($applied[$key])) {
- unset($this->patches[$key]);
+ unset($patches[$key]);
continue;
}
if ($apply_only && $apply_only != $key) {
- unset($this->patches[$key]);
+ unset($patches[$key]);
continue;
}
$can_apply = true;
foreach ($patch->getAfter() as $after) {
if (empty($applied[$after])) {
if ($apply_only) {
echo pht(
"Unable to apply patch '%s' because it depends ".
"on patch '%s', which has not been applied.\n",
$apply_only,
$after);
return 1;
}
$can_apply = false;
break;
}
}
if (!$can_apply) {
continue;
}
$applied_something = true;
if ($this->dryRun) {
echo pht("DRYRUN: Would apply patch '%s'.", $key)."\n";
} else {
echo pht("Applying patch '%s'...", $key)."\n";
$t_begin = microtime(true);
$api->applyPatch($patch);
$t_end = microtime(true);
if (!$skip_mark) {
$api->markPatchApplied($key, ($t_end - $t_begin));
}
}
- unset($this->patches[$key]);
+ unset($patches[$key]);
$applied[$key] = true;
}
if (!$applied_something) {
- if (count($this->patches)) {
+ if (count($patches)) {
throw new Exception(
pht(
'Some patches could not be applied to "%s": %s',
$api->getRef()->getRefKey(),
- implode(', ', array_keys($this->patches))));
+ implode(', ', array_keys($patches))));
} else if (!$this->dryRun && !$apply_only) {
echo pht(
'Storage is up to date on "%s". Use "%s" for details.',
$api->getRef()->getRefKey(),
'storage status')."\n";
}
break;
}
}
}
final protected function getBareHostAndPort($host) {
// Split out port information, since the command-line client requires a
// separate flag for the port.
$uri = new PhutilURI('mysql://'.$host);
if ($uri->getPort()) {
$port = $uri->getPort();
$bare_hostname = $uri->getDomain();
} else {
$port = null;
$bare_hostname = $host;
}
return array($bare_hostname, $port);
}
/**
* Acquires a @{class:PhabricatorGlobalLock}.
*
* @return PhabricatorGlobalLock
*/
final protected function lock(PhabricatorStorageManagementAPI $api) {
return PhabricatorGlobalLock::newLock(__CLASS__)
->useSpecificConnection($api->getConn(null))
->lock();
}
}

File Metadata

Mime Type
text/x-diff
Expires
Thu, Nov 6, 4:48 AM (19 h, 14 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
321620
Default Alt Text
(56 KB)

Event Timeline