No OneTemporary
Actions

Size

73 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/scripts/sql/manage_storage.php b/scripts/sql/manage_storage.php
	index 64a50dcdb0..d1e1d16b6f 100755
	--- a/scripts/sql/manage_storage.php
	+++ b/scripts/sql/manage_storage.php
	@@ -1,176 +1,172 @@
	#!/usr/bin/env php
	<?php

	$root = dirname(dirname(dirname(__FILE__)));
	require_once $root.'/scripts/__init_script__.php';

	$args = new PhutilArgumentParser($argv);
	$args->setTagline(pht('manage Phabricator storage and schemata'));
	$args->setSynopsis(<<<EOHELP
	storage __workflow__ [__options__]
	Manage Phabricator database storage and schema versioning.

	storage upgrade
	Initialize or upgrade Phabricator storage.

	storage upgrade --user __root__ --password __hunter2__
	Use administrative credentials for schema changes.
	EOHELP
	);
	$args->parseStandardArguments();

	$conf = PhabricatorEnv::newObjectFromConfig(
	'mysql.configuration-provider',
	array($dao = null, 'w'));

	$default_user = $conf->getUser();
	$default_host = $conf->getHost();
	$default_port = $conf->getPort();
	$default_namespace = PhabricatorLiskDAO::getDefaultStorageNamespace();

	try {
	$args->parsePartial(
	array(
	array(
	'name' => 'force',
	'short' => 'f',
	'help' => pht(
	'Do not prompt before performing dangerous operations.'),
	),
	array(
	'name' => 'user',
	'short' => 'u',
	'param' => 'username',
	'default' => $default_user,
	'help' => pht(
	"Connect with __username__ instead of the configured default ('%s').",
	$default_user),
	),
	array(
	'name' => 'password',
	'short' => 'p',
	'param' => 'password',
	'help' => pht('Use __password__ instead of the configured default.'),
	),
	array(
	'name' => 'namespace',
	'param' => 'name',
	'default' => $default_namespace,
	'help' => pht(
	"Use namespace __namespace__ instead of the configured ".
	"default ('%s'). This is an advanced feature used by unit tests; ".
	"you should not normally use this flag.",
	$default_namespace),
	),
	array(
	- 'name' => 'dryrun',
	- 'help' => pht(
	+ 'name' => 'dryrun',
	+ 'help' => pht(
	'Do not actually change anything, just show what would be changed.'),
	),
	array(
	- 'name' => 'disable-utf8mb4',
	- 'help' => pht(
	- 'Disable utf8mb4, even if the database supports it. This is an '.
	+ 'name' => 'disable-utf8mb4',
	+ 'help' => pht(
	+ 'Disable %s, even if the database supports it. This is an '.
	'advanced feature used for testing changes to Phabricator; you '.
	- 'should not normally use this flag.'),
	+ 'should not normally use this flag.',
	+ 'utf8mb4'),
	),
	));
	} catch (PhutilArgumentUsageException $ex) {
	$args->printUsageException($ex);
	exit(77);
	}

	// First, test that the Phabricator configuration is set up correctly. After
	// we know this works we'll test any administrative credentials specifically.

	-$test_api = new PhabricatorStorageManagementAPI();
	-$test_api->setUser($default_user);
	-$test_api->setHost($default_host);
	-$test_api->setPort($default_port);
	-$test_api->setPassword($conf->getPassword());
	-$test_api->setNamespace($args->getArg('namespace'));
	+$test_api = id(new PhabricatorStorageManagementAPI())
	+ ->setUser($default_user)
	+ ->setHost($default_host)
	+ ->setPort($default_port)
	+ ->setPassword($conf->getPassword())
	+ ->setNamespace($args->getArg('namespace'));

	try {
	queryfx(
	$test_api->getConn(null),
	'SELECT 1');
	} catch (AphrontQueryException $ex) {
	$message = phutil_console_format(
	"%s\n\n%s\n\n%s\n\n%s\n\n%s: %s\n",
	pht('MySQL Credentials Not Configured'),
	pht(
	'Unable to connect to MySQL using the configured credentials. '.
	'You must configure standard credentials before you can upgrade '.
	'storage. Run these commands to set up credentials:'),
	" phabricator/ $ ./bin/config set mysql.host __host__\n".
	" phabricator/ $ ./bin/config set mysql.user __username__\n".
	" phabricator/ $ ./bin/config set mysql.pass __password__",
	pht(
	'These standard credentials are separate from any administrative '.
	'credentials provided to this command with __%s__ or '.
	'__%s__, and must be configured correctly before you can proceed.',
	'--user',
	'--password'),
	pht('Raw MySQL Error'),
	$ex->getMessage());
	-
	echo phutil_console_wrap($message);
	-
	exit(1);
	}

	-
	if ($args->getArg('password') === null) {
	// This is already a PhutilOpaqueEnvelope.
	$password = $conf->getPassword();
	} else {
	// Put this in a PhutilOpaqueEnvelope.
	$password = new PhutilOpaqueEnvelope($args->getArg('password'));
	PhabricatorEnv::overrideConfig('mysql.pass', $args->getArg('password'));
	}

	-$api = new PhabricatorStorageManagementAPI();
	-$api->setUser($args->getArg('user'));
	-PhabricatorEnv::overrideConfig('mysql.user', $args->getArg('user'));
	-$api->setHost($default_host);
	-$api->setPort($default_port);
	-$api->setPassword($password);
	-$api->setNamespace($args->getArg('namespace'));
	-$api->setDisableUTF8MB4($args->getArg('disable-utf8mb4'));
	+$api = id(new PhabricatorStorageManagementAPI())
	+ ->setUser($args->getArg('user'))
	+ ->setHost($default_host)
	+ ->setPort($default_port)
	+ ->setPassword($password)
	+ ->setNamespace($args->getArg('namespace'))
	+ ->setDisableUTF8MB4($args->getArg('disable-utf8mb4'));
	+PhabricatorEnv::overrideConfig('mysql.user', $api->getUser());

	try {
	queryfx(
	$api->getConn(null),
	'SELECT 1');
	} catch (AphrontQueryException $ex) {
	$message = phutil_console_format(
	"%s\n\n%s\n\n%s: %s\n",
	pht('Bad Administrative Credentials'),
	pht(
	'Unable to connect to MySQL using the administrative credentials '.
	'provided with the __%s__ and __%s__ flags. Check that '.
	'you have entered them correctly.',
	'--user',
	'--password'),
	pht('Raw MySQL Error'),
	$ex->getMessage());
	-
	echo phutil_console_wrap($message);
	-
	exit(1);
	}

	$workflows = id(new PhutilClassMapQuery())
	->setAncestorClass('PhabricatorStorageManagementWorkflow')
	->execute();

	$patches = PhabricatorSQLPatchList::buildAllPatches();

	foreach ($workflows as $workflow) {
	$workflow->setAPI($api);
	$workflow->setPatches($patches);
	}

	$workflows[] = new PhutilHelpArgumentWorkflow();

	$args->parseWorkflows($workflows);
	diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php
	index e7437a30b9..465f8728cc 100644
	--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php
	+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php
	@@ -1,66 +1,64 @@
	<?php

	final class PhabricatorStorageManagementAdjustWorkflow
	extends PhabricatorStorageManagementWorkflow {

	protected function didConstruct() {
	$this
	->setName('adjust')
	->setExamples('adjust [__options__]')
	->setSynopsis(
	pht(
	'Make schemata adjustments to correct issues with characters sets, '.
	'collations, and keys.'))
	->setArguments(
	array(
	array(
	'name' => 'unsafe',
	'help' => pht(
	'Permit adjustments which truncate data. This option may '.
	'destroy some data, but the lost data is usually not '.
	'important (most commonly, the ends of very long object '.
	'titles).'),
	),
	));
	}

	- public function execute(PhutilArgumentParser $args) {
	- $force = $args->getArg('force');
	+ public function didExecute(PhutilArgumentParser $args) {
	$unsafe = $args->getArg('unsafe');
	- $dry_run = $args->getArg('dryrun');

	$this->requireAllPatchesApplied();
	- return $this->adjustSchemata($force, $unsafe, $dry_run);
	+ return $this->adjustSchemata($unsafe);
	}

	private function requireAllPatchesApplied() {
	$api = $this->getAPI();
	$applied = $api->getAppliedPatches();

	if ($applied === null) {
	throw new PhutilArgumentUsageException(
	pht(
	'You have not initialized the database yet. You must initialize '.
	'the database before you can adjust schemata. Run `%s` '.
	'to initialize the database.',
	'storage upgrade'));
	}

	$applied = array_fuse($applied);

	$patches = $this->getPatches();
	$patches = mpull($patches, null, 'getFullKey');
	$missing = array_diff_key($patches, $applied);

	if ($missing) {
	throw new PhutilArgumentUsageException(
	pht(
	'You have not applied all available storage patches yet. You must '.
	'apply all available patches before you can adjust schemata. '.
	'Run `%s` to show patch status, and `%s` to apply missing patches.',
	'storage status',
	'storage upgrade'));
	}
	}

	}
	diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDatabasesWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDatabasesWorkflow.php
	index 5509a32d73..b10e9b9e41 100644
	--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDatabasesWorkflow.php
	+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDatabasesWorkflow.php
	@@ -1,23 +1,22 @@
	<?php

	final class PhabricatorStorageManagementDatabasesWorkflow
	extends PhabricatorStorageManagementWorkflow {

	protected function didConstruct() {
	$this
	->setName('databases')
	->setExamples('databases [__options__]')
	->setSynopsis(pht('List Phabricator databases.'));
	}

	- public function execute(PhutilArgumentParser $args) {
	- $api = $this->getAPI();
	+ public function didExecute(PhutilArgumentParser $args) {
	+ $api = $this->getAPI();
	$patches = $this->getPatches();

	- $databases = $api->getDatabaseList($patches, $only_living = true);
	+ $databases = $api->getDatabaseList($patches, true);
	echo implode("\n", $databases)."\n";
	-
	return 0;
	}

	}
	diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php
	index 6c1c216359..9bf5906148 100644
	--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php
	+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDestroyWorkflow.php
	@@ -1,84 +1,89 @@
	<?php

	final class PhabricatorStorageManagementDestroyWorkflow
	extends PhabricatorStorageManagementWorkflow {

	protected function didConstruct() {
	$this
	->setName('destroy')
	->setExamples('destroy [__options__]')
	->setSynopsis(pht('Permanently destroy all storage and data.'))
	->setArguments(
	array(
	array(
	'name' => 'unittest-fixtures',
	'help' => pht(
	'Restrict destroy operations to databases created '.
	'by %s test fixtures.',
	'PhabricatorTestCase'),
	),
	));
	}

	- public function execute(PhutilArgumentParser $args) {
	- $is_dry = $args->getArg('dryrun');
	- $is_force = $args->getArg('force');
	+ public function didExecute(PhutilArgumentParser $args) {
	+ $console = PhutilConsole::getConsole();

	- if (!$is_dry && !$is_force) {
	- echo phutil_console_wrap(
	- pht(
	- 'Are you completely sure you really want to permanently destroy all '.
	- 'storage for Phabricator data? This operation can not be undone and '.
	- 'your data will not be recoverable if you proceed.'));
	+ if (!$this->isDryRun() && !$this->isForce()) {
	+ $console->writeOut(
	+ phutil_console_wrap(
	+ pht(
	+ 'Are you completely sure you really want to permanently destroy '.
	+ 'all storage for Phabricator data? This operation can not be '.
	+ 'undone and your data will not be recoverable if you proceed.')));

	if (!phutil_console_confirm(pht('Permanently destroy all data?'))) {
	- echo pht('Cancelled.')."\n";
	+ $console->writeOut("%s\n", pht('Cancelled.'));
	exit(1);
	}

	if (!phutil_console_confirm(pht('Really destroy all data forever?'))) {
	- echo pht('Cancelled.')."\n";
	+ $console->writeOut("%s\n", pht('Cancelled.'));
	exit(1);
	}
	}

	- $api = $this->getAPI();
	+ $api = $this->getAPI();
	$patches = $this->getPatches();

	if ($args->getArg('unittest-fixtures')) {
	$conn = $api->getConn(null);
	$databases = queryfx_all(
	$conn,
	'SELECT DISTINCT(TABLE_SCHEMA) AS db '.
	'FROM INFORMATION_SCHEMA.TABLES '.
	'WHERE TABLE_SCHEMA LIKE %>',
	PhabricatorTestCase::NAMESPACE_PREFIX);
	$databases = ipull($databases, 'db');
	} else {
	- $databases = $api->getDatabaseList($patches);
	+ $databases = $api->getDatabaseList($patches);
	$databases[] = $api->getDatabaseName('meta_data');
	+
	// These are legacy databases that were dropped long ago. See T2237.
	$databases[] = $api->getDatabaseName('phid');
	$databases[] = $api->getDatabaseName('directory');
	}

	foreach ($databases as $database) {
	- if ($is_dry) {
	- echo pht("DRYRUN: Would drop database '%s'.", $database)."\n";
	+ if ($this->isDryRun()) {
	+ $console->writeOut(
	+ "%s\n",
	+ pht("DRYRUN: Would drop database '%s'.", $database));
	} else {
	- echo pht("Dropping database '%s'...", $database)."\n";
	+ $console->writeOut(
	+ "%s\n",
	+ pht("Dropping database '%s'...", $database));
	queryfx(
	$api->getConn(null),
	'DROP DATABASE IF EXISTS %T',
	$database);
	}
	}

	- if (!$is_dry) {
	- echo pht('Storage was destroyed.')."\n";
	+ if (!$this->isDryRun()) {
	+ $console->writeOut("%s\n", pht('Storage was destroyed.'));
	}

	return 0;
	}

	}
	diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php
	index e9a09bccd1..38ce117a08 100644
	--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php
	+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php
	@@ -1,57 +1,58 @@
	<?php

	final class PhabricatorStorageManagementDumpWorkflow
	extends PhabricatorStorageManagementWorkflow {

	protected function didConstruct() {
	$this
	->setName('dump')
	->setExamples('dump [__options__]')
	->setSynopsis(pht('Dump all data in storage to stdout.'));
	}

	- public function execute(PhutilArgumentParser $args) {
	- $console = PhutilConsole::getConsole();
	- $api = $this->getAPI();
	+ public function didExecute(PhutilArgumentParser $args) {
	+ $api = $this->getAPI();
	$patches = $this->getPatches();

	+ $console = PhutilConsole::getConsole();
	+
	$applied = $api->getAppliedPatches();
	if ($applied === null) {
	$namespace = $api->getNamespace();
	$console->writeErr(
	pht(
	'Storage Not Initialized: There is no database storage '.
	'initialized in this storage namespace ("%s"). Use '.
	'%s to initialize storage.',
	$namespace,
	- 'storage upgrade'));
	+ './bin/storage upgrade'));
	return 1;
	}

	- $databases = $api->getDatabaseList($patches, $only_living = true);
	+ $databases = $api->getDatabaseList($patches, true);

	list($host, $port) = $this->getBareHostAndPort($api->getHost());

	$flag_password = '';
	$password = $api->getPassword();
	if ($password) {
	if (strlen($password->openEnvelope())) {
	$flag_password = csprintf('-p%P', $password);
	}
	}

	$flag_port = $port
	? csprintf('--port %d', $port)
	: '';

	return phutil_passthru(
	'mysqldump --hex-blob --single-transaction --default-character-set=utf8 '.
	'-u %s %C -h %s %C --databases %Ls',
	$api->getUser(),
	$flag_password,
	$host,
	$flag_port,
	$databases);
	}

	}
	diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementProbeWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementProbeWorkflow.php
	index 8e373d0cf6..d65b9bd85a 100644
	--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementProbeWorkflow.php
	+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementProbeWorkflow.php
	@@ -1,101 +1,101 @@
	<?php

	final class PhabricatorStorageManagementProbeWorkflow
	extends PhabricatorStorageManagementWorkflow {

	protected function didConstruct() {
	$this
	->setName('probe')
	->setExamples('probe')
	->setSynopsis(pht('Show approximate table sizes.'));
	}

	- public function execute(PhutilArgumentParser $args) {
	+ public function didExecute(PhutilArgumentParser $args) {
	$console = PhutilConsole::getConsole();
	$console->writeErr(
	"%s\n",
	pht('Analyzing table sizes (this may take a moment)...'));

	- $api = $this->getAPI();
	- $patches = $this->getPatches();
	- $databases = $api->getDatabaseList($patches, $only_living = true);
	+ $api = $this->getAPI();
	+ $patches = $this->getPatches();
	+ $databases = $api->getDatabaseList($patches, true);

	$conn_r = $api->getConn(null);

	$data = array();
	foreach ($databases as $database) {
	queryfx($conn_r, 'USE %C', $database);
	$tables = queryfx_all(
	$conn_r,
	'SHOW TABLE STATUS');
	$tables = ipull($tables, null, 'Name');
	$data[$database] = $tables;
	}

	$totals = array_fill_keys(array_keys($data), 0);
	$overall = 0;

	foreach ($data as $db => $tables) {
	foreach ($tables as $table => $info) {
	$table_size = $info['Data_length'] + $info['Index_length'];

	$data[$db][$table]['_totalSize'] = $table_size;
	$totals[$db] += $table_size;
	$overall += $table_size;
	}
	}

	asort($totals);

	$table = id(new PhutilConsoleTable())
	->setShowHeader(false)
	->setPadding(2)
	->addColumn('name', array('title' => pht('Database / Table')))
	->addColumn('size', array('title' => pht('Size')))
	->addColumn('percentage', array('title' => pht('Percentage')));

	foreach ($totals as $db => $size) {
	list($database_size, $database_percentage) = $this->formatSize(
	$totals[$db],
	$overall);

	$table->addRow(array(
	'name' => tsprintf('%s', $db),
	'size' => tsprintf('%s', $database_size),
	'percentage' => tsprintf('%s', $database_percentage),
	));
	$data[$db] = isort($data[$db], '_totalSize');
	foreach ($data[$db] as $table_name => $info) {
	list($table_size, $table_percentage) = $this->formatSize(
	$info['_totalSize'],
	$overall);

	$table->addRow(array(
	'name' => ' '.$table_name,
	'size' => $table_size,
	'percentage' => $table_percentage,
	));
	}
	}

	list($overall_size, $overall_percentage) = $this->formatSize(
	$overall,
	$overall);
	$table->addRow(array(
	'name' => tsprintf('%s', pht('TOTAL')),
	'size' => tsprintf('%s', $overall_size),
	'percentage' => tsprintf('%s', $overall_percentage),
	));

	$table->draw();
	return 0;
	}

	private function formatSize($n, $o) {
	return array(
	sprintf('%8.8s MB', number_format($n / (1024 * 1024), 1)),
	sprintf('%3.1f%%', 100 * ($n / $o)),
	);
	}

	}
	diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php
	index 561e01cedd..2d67d97e48 100644
	--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php
	+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php
	@@ -1,165 +1,169 @@
	<?php

	final class PhabricatorStorageManagementQuickstartWorkflow
	extends PhabricatorStorageManagementWorkflow {

	protected function didConstruct() {
	$this
	->setName('quickstart')
	->setExamples('quickstart [__options__]')
	->setSynopsis(
	pht(
	'Generate a new quickstart database dump. This command is mostly '.
	'useful when developing Phabricator.'))
	->setArguments(
	array(
	array(
	'name' => 'output',
	'param' => 'file',
	'help' => pht('Specify output file to write.'),
	),
	));
	}

	public function execute(PhutilArgumentParser $args) {
	+ parent::execute($args);
	+
	$output = $args->getArg('output');
	if (!$output) {
	throw new PhutilArgumentUsageException(
	pht(
	'Specify a file to write with `%s`.',
	'--output'));
	}

	$namespace = 'phabricator_quickstart_'.Filesystem::readRandomCharacters(8);

	$bin = dirname(phutil_get_library_root('phabricator')).'/bin/storage';

	if (!$this->getAPI()->isCharacterSetAvailable('utf8mb4')) {
	throw new PhutilArgumentUsageException(
	pht(
	'You can only generate a new quickstart file if MySQL supports '.
	- 'the utf8mb4 character set (available in MySQL 5.5 and newer). The '.
	- 'configured server does not support utf8mb4.'));
	+ 'the %s character set (available in MySQL 5.5 and newer). The '.
	+ 'configured server does not support %s.',
	+ 'utf8mb4',
	+ 'utf8mb4'));
	}

	$err = phutil_passthru(
	'%s upgrade --force --no-quickstart --namespace %s',
	$bin,
	$namespace);
	if ($err) {
	return $err;
	}

	$err = phutil_passthru(
	'%s adjust --force --namespace %s',
	$bin,
	$namespace);
	if ($err) {
	return $err;
	}

	$tmp = new TempFile();
	$err = phutil_passthru(
	'%s dump --namespace %s > %s',
	$bin,
	$namespace,
	$tmp);
	if ($err) {
	return $err;
	}

	$err = phutil_passthru(
	'%s destroy --force --namespace %s',
	$bin,
	$namespace);
	if ($err) {
	return $err;
	}

	$dump = Filesystem::readFile($tmp);

	$dump = str_replace(
	$namespace,
	'{$NAMESPACE}',
	$dump);

	// NOTE: This is a hack. We can not use `binary` for these columns, because
	// they are a part of a fulltext index. This regex is avoiding matching a
	// possible NOT NULL at the end of the line.
	$old = $dump;
	$dump = preg_replace(
	'/`corpus` longtext CHARACTER SET .*? COLLATE [^\s,]+/mi',
	'`corpus` longtext CHARACTER SET {$CHARSET_FULLTEXT} '.
	'COLLATE {$COLLATE_FULLTEXT}',
	$dump);
	if ($dump == $old) {
	// If we didn't make any changes, yell about it. We'll produce an invalid
	// dump otherwise.
	throw new PhutilArgumentUsageException(
	pht(
	'Failed to apply hack to adjust %s search column!',
	'FULLTEXT'));
	}

	$dump = str_replace(
	'utf8mb4_bin',
	'{$COLLATE_TEXT}',
	$dump);

	$dump = str_replace(
	'utf8mb4_unicode_ci',
	'{$COLLATE_SORT}',
	$dump);

	$dump = str_replace(
	'utf8mb4',
	'{$CHARSET}',
	$dump);

	$old = $dump;
	$dump = preg_replace(
	'/CHARACTER SET {\$CHARSET} COLLATE {\$COLLATE_SORT}/mi',
	'CHARACTER SET {$CHARSET_SORT} COLLATE {$COLLATE_SORT}',
	$dump);
	if ($dump == $old) {
	throw new PhutilArgumentUsageException(
	pht('Failed to adjust SORT columns!'));
	}

	// Strip out a bunch of unnecessary commands which make the dump harder
	// to handle and slower to import.

	// Remove character set adjustments and key disables.
	$dump = preg_replace(
	'(^/\.\*/;$)m',
	'',
	$dump);

	// Remove comments.
	$dump = preg_replace('/^--.*$/m', '', $dump);

	// Remove table drops, locks, and unlocks. These are never relevant when
	- // performing q quickstart.
	+ // performing a quickstart.
	$dump = preg_replace(
	'/^(DROP TABLE\|LOCK TABLES\|UNLOCK TABLES).*$/m',
	'',
	$dump);

	// Collapse adjacent newlines.
	$dump = preg_replace('/\n\s*\n/', "\n", $dump);

	$dump = str_replace(';', ";\n", $dump);
	$dump = trim($dump)."\n";

	Filesystem::writeFile($output, $dump);

	$console = PhutilConsole::getConsole();
	$console->writeOut(
	"<bg:green> %s </bg> %s\n",
	pht('SUCCESS'),
	pht('Wrote fresh quickstart SQL.'));

	return 0;
	}

	}
	diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementRenamespaceWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementRenamespaceWorkflow.php
	index 9de4499b23..ad4960ceba 100644
	--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementRenamespaceWorkflow.php
	+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementRenamespaceWorkflow.php
	@@ -1,94 +1,100 @@
	<?php

	final class PhabricatorStorageManagementRenamespaceWorkflow
	extends PhabricatorStorageManagementWorkflow {

	protected function didConstruct() {
	$this
	->setName('renamespace')
	->setExamples(
	'renamespace [__options__] '.
	'--in __dump.sql__ --from __old__ --to __new__ > __out.sql__')
	->setSynopsis(pht('Change the database namespace of a .sql dump file.'))
	->setArguments(
	array(
	array(
	'name' => 'in',
	'param' => 'file',
	'help' => pht('SQL dumpfile to process.'),
	),
	array(
	'name' => 'from',
	'param' => 'namespace',
	'help' => pht('Current database namespace used by dumpfile.'),
	),
	array(
	'name' => 'to',
	'param' => 'namespace',
	'help' => pht('Desired database namespace for output.'),
	),
	));
	}

	- public function execute(PhutilArgumentParser $args) {
	+ public function didExecute(PhutilArgumentParser $args) {
	$console = PhutilConsole::getConsole();

	$in = $args->getArg('in');
	if (!strlen($in)) {
	throw new PhutilArgumentUsageException(
	- pht('Specify the dumpfile to read with --in.'));
	+ pht(
	+ 'Specify the dumpfile to read with %s.',
	+ '--in'));
	}

	$from = $args->getArg('from');
	if (!strlen($from)) {
	throw new PhutilArgumentUsageException(
	- pht('Specify namespace to rename from with --from.'));
	+ pht(
	+ 'Specify namespace to rename from with %s.',
	+ '--from'));
	}

	$to = $args->getArg('to');
	if (!strlen($to)) {
	throw new PhutilArgumentUsageException(
	- pht('Specify namespace to rename to with --to.'));
	+ pht(
	+ 'Specify namespace to rename to with %s.',
	+ '--to'));
	}

	$patterns = array(
	'use' => '@^(USE `)([^_]+)(_.*)$@',
	'create' => '@^(CREATE DATABASE /\.?\/ `)([^_]+)(_.)$@',
	);

	$found = array_fill_keys(array_keys($patterns), 0);

	$matches = null;
	foreach (new LinesOfALargeFile($in) as $line) {

	foreach ($patterns as $key => $pattern) {
	if (preg_match($pattern, $line, $matches)) {
	$namespace = $matches[2];
	if ($namespace != $from) {
	throw new Exception(
	pht(
	'Expected namespace "%s", found "%s": %s.',
	$from,
	$namespace,
	$line));
	}

	$line = $matches[1].$to.$matches[3];
	$found[$key]++;
	}
	}

	echo $line."\n";
	}

	// Give the user a chance to catch things if the results are crazy.
	$console->writeErr(
	pht(
	'Adjusted %s create statements and %s use statements.',
	new PhutilNumber($found['create']),
	new PhutilNumber($found['use']))."\n");

	return 0;
	}

	}
	diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementShellWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementShellWorkflow.php
	index 58a7c072a3..c66fc73e98 100644
	--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementShellWorkflow.php
	+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementShellWorkflow.php
	@@ -1,38 +1,40 @@
	<?php

	final class PhabricatorStorageManagementShellWorkflow
	extends PhabricatorStorageManagementWorkflow {

	protected function didConstruct() {
	$this
	->setName('shell')
	->setExamples('shell [__options__]')
	->setSynopsis(pht('Launch an interactive shell.'));
	}

	public function execute(PhutilArgumentParser $args) {
	+
	+
	$api = $this->getAPI();
	list($host, $port) = $this->getBareHostAndPort($api->getHost());

	$flag_port = $port
	? csprintf('--port %d', $port)
	: '';

	$flag_password = '';
	$password = $api->getPassword();
	if ($password) {
	if (strlen($password->openEnvelope())) {
	$flag_password = csprintf('--password=%P', $password);
	}
	}

	return phutil_passthru(
	'mysql --default-character-set=utf8 '.
	'-u %s %C -h %s %C',
	$api->getUser(),
	$flag_password,
	$host,
	$flag_port);
	}

	}
	diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementStatusWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementStatusWorkflow.php
	index 774ae8445c..be4f8e5434 100644
	--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementStatusWorkflow.php
	+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementStatusWorkflow.php
	@@ -1,61 +1,61 @@
	<?php

	final class PhabricatorStorageManagementStatusWorkflow
	extends PhabricatorStorageManagementWorkflow {

	protected function didConstruct() {
	$this
	->setName('status')
	->setExamples('status [__options__]')
	->setSynopsis(pht('Show patch application status.'));
	}

	- public function execute(PhutilArgumentParser $args) {
	- $api = $this->getAPI();
	+ public function didExecute(PhutilArgumentParser $args) {
	+ $api = $this->getAPI();
	$patches = $this->getPatches();

	$applied = $api->getAppliedPatches();

	if ($applied === null) {
	echo phutil_console_format(
	"%s: %s\n",
	pht('Database Not Initialized'),
	- pht('Run %s to initialize.', 'storage upgrade'));
	+ pht('Run %s to initialize.', './bin/storage upgrade'));

	return 1;
	}

	$table = id(new PhutilConsoleTable())
	->setShowHeader(false)
	- ->addColumn('id', array('title' => pht('ID')))
	- ->addColumn('status', array('title' => pht('Status')))
	+ ->addColumn('id', array('title' => pht('ID')))
	+ ->addColumn('status', array('title' => pht('Status')))
	->addColumn('duration', array('title' => pht('Duration')))
	- ->addColumn('type', array('title' => pht('Type')))
	- ->addColumn('name', array('title' => pht('Name')));
	+ ->addColumn('type', array('title' => pht('Type')))
	+ ->addColumn('name', array('title' => pht('Name')));

	$durations = $api->getPatchDurations();

	foreach ($patches as $patch) {
	$duration = idx($durations, $patch->getFullKey());
	if ($duration === null) {
	$duration = '-';
	} else {
	$duration = pht('%s us', new PhutilNumber($duration));
	}

	$table->addRow(array(
	'id' => $patch->getFullKey(),
	'status' => in_array($patch->getFullKey(), $applied)
	? pht('Applied')
	: pht('Not Applied'),
	'duration' => $duration,
	'type' => $patch->getType(),
	'name' => $patch->getName(),
	));
	}

	$table->draw();
	return 0;
	}

	}
	diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php
	index d5a4f41cad..70dc90fdf7 100644
	--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php
	+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php
	@@ -1,230 +1,88 @@
	<?php

	final class PhabricatorStorageManagementUpgradeWorkflow
	extends PhabricatorStorageManagementWorkflow {

	protected function didConstruct() {
	$this
	->setName('upgrade')
	->setExamples('upgrade [__options__]')
	->setSynopsis(pht('Upgrade database schemata.'))
	->setArguments(
	array(
	array(
	'name' => 'apply',
	'param' => 'patch',
	'help' => pht(
	'Apply __patch__ explicitly. This is an advanced feature for '.
	'development and debugging; you should not normally use this '.
	'flag. This skips adjustment.'),
	),
	array(
	'name' => 'no-quickstart',
	'help' => pht(
	- 'Build storage patch-by-patch from scatch, even if it could '.
	+ 'Build storage patch-by-patch from scratch, even if it could '.
	'be loaded from the quickstart template.'),
	),
	array(
	'name' => 'init-only',
	'help' => pht(
	'Initialize storage only; do not apply patches or adjustments.'),
	),
	array(
	'name' => 'no-adjust',
	'help' => pht(
	'Do not apply storage adjustments after storage upgrades.'),
	),
	));
	}

	- public function execute(PhutilArgumentParser $args) {
	- $is_dry = $args->getArg('dryrun');
	- $is_force = $args->getArg('force');
	-
	- $api = $this->getAPI();
	+ public function didExecute(PhutilArgumentParser $args) {
	+ $console = PhutilConsole::getConsole();
	$patches = $this->getPatches();

	- if (!$is_dry && !$is_force) {
	- echo phutil_console_wrap(
	- pht(
	- 'Before running storage upgrades, you should take down the '.
	- 'Phabricator web interface and stop any running Phabricator '.
	- 'daemons (you can disable this warning with %s).',
	- '--force'));
	+ if (!$this->isDryRun() && !$this->isForce()) {
	+ $console->writeOut(
	+ phutil_console_wrap(
	+ pht(
	+ 'Before running storage upgrades, you should take down the '.
	+ 'Phabricator web interface and stop any running Phabricator '.
	+ 'daemons (you can disable this warning with %s).',
	+ '--force')));

	if (!phutil_console_confirm(pht('Are you ready to continue?'))) {
	- echo pht('Cancelled.')."\n";
	+ $console->writeOut("%s\n", pht('Cancelled.'));
	return 1;
	}
	}

	$apply_only = $args->getArg('apply');
	if ($apply_only) {
	if (empty($patches[$apply_only])) {
	throw new PhutilArgumentUsageException(
	pht(
	"%s argument '%s' is not a valid patch. ".
	"Use '%s' to show patch status.",
	'--apply',
	$apply_only,
	- 'storage status'));
	+ './bin/storage status'));
	}
	}

	$no_quickstart = $args->getArg('no-quickstart');
	- $init_only = $args->getArg('init-only');
	- $no_adjust = $args->getArg('no-adjust');
	-
	- $applied = $api->getAppliedPatches();
	- if ($applied === null) {
	-
	- if ($is_dry) {
	- echo pht(
	- "DRYRUN: Patch metadata storage doesn't exist yet, ".
	- "it would be created.\n");
	- return 0;
	- }
	-
	- if ($apply_only) {
	- throw new PhutilArgumentUsageException(
	- pht(
	- 'Storage has not been initialized yet, you must initialize '.
	- 'storage before selectively applying patches.'));
	- return 1;
	- }
	-
	- $legacy = $api->getLegacyPatches($patches);
	- if ($legacy \|\| $no_quickstart \|\| $init_only) {
	-
	- // If we have legacy patches, we can't quickstart.
	-
	- $api->createDatabase('meta_data');
	- $api->createTable(
	- 'meta_data',
	- 'patch_status',
	- array(
	- 'patch VARCHAR(255) NOT NULL PRIMARY KEY COLLATE utf8_general_ci',
	- 'applied INT UNSIGNED NOT NULL',
	- ));
	-
	- foreach ($legacy as $patch) {
	- $api->markPatchApplied($patch);
	- }
	- } else {
	- echo pht('Loading quickstart template...')."\n";
	- $root = dirname(phutil_get_library_root('phabricator'));
	- $sql = $root.'/resources/sql/quickstart.sql';
	- $api->applyPatchSQL($sql);
	- }
	- }
	-
	- if ($init_only) {
	- echo pht('Storage initialized.')."\n";
	- return 0;
	- }
	-
	- $applied = $api->getAppliedPatches();
	- $applied = array_fuse($applied);
	-
	- $skip_mark = false;
	- if ($apply_only) {
	- if (isset($applied[$apply_only])) {
	-
	- unset($applied[$apply_only]);
	- $skip_mark = true;
	-
	- if (!$is_force && !$is_dry) {
	- echo phutil_console_wrap(
	- pht(
	- "Patch '%s' has already been applied. Are you sure you want ".
	- "to apply it again? This may put your storage in a state ".
	- "that the upgrade scripts can not automatically manage.",
	- $apply_only));
	- if (!phutil_console_confirm(pht('Apply patch again?'))) {
	- echo pht('Cancelled.')."\n";
	- return 1;
	- }
	- }
	- }
	- }
	-
	- while (true) {
	- $applied_something = false;
	- foreach ($patches as $key => $patch) {
	- if (isset($applied[$key])) {
	- unset($patches[$key]);
	- continue;
	- }
	-
	- if ($apply_only && $apply_only != $key) {
	- unset($patches[$key]);
	- continue;
	- }
	-
	- $can_apply = true;
	- foreach ($patch->getAfter() as $after) {
	- if (empty($applied[$after])) {
	- if ($apply_only) {
	- echo pht(
	- "Unable to apply patch '%s' because it depends ".
	- "on patch '%s', which has not been applied.\n",
	- $apply_only,
	- $after);
	- return 1;
	- }
	- $can_apply = false;
	- break;
	- }
	- }
	+ $init_only = $args->getArg('init-only');
	+ $no_adjust = $args->getArg('no-adjust');

	- if (!$can_apply) {
	- continue;
	- }
	+ $this->upgradeSchemata($apply_only, $no_quickstart, $init_only);

	- $applied_something = true;
	-
	- if ($is_dry) {
	- echo pht("DRYRUN: Would apply patch '%s'.", $key)."\n";
	- } else {
	- echo pht("Applying patch '%s'...", $key)."\n";
	-
	- $t_begin = microtime(true);
	- $api->applyPatch($patch);
	- $t_end = microtime(true);
	-
	- if (!$skip_mark) {
	- $api->markPatchApplied($key, ($t_end - $t_begin));
	- }
	- }
	-
	- unset($patches[$key]);
	- $applied[$key] = true;
	- }
	-
	- if (!$applied_something) {
	- if (count($patches)) {
	- throw new Exception(
	- pht(
	- 'Some patches could not be applied: %s',
	- implode(', ', array_keys($patches))));
	- } else if (!$is_dry && !$apply_only) {
	- echo pht(
	- "Storage is up to date. Use '%s' for details.",
	- 'storage status')."\n";
	- }
	- break;
	- }
	- }
	-
	- $console = PhutilConsole::getConsole();
	if ($no_adjust \|\| $init_only \|\| $apply_only) {
	$console->writeOut(
	"%s\n",
	pht('Declining to apply storage adjustments.'));
	return 0;
	} else {
	- return $this->adjustSchemata($is_force, $unsafe = false, $is_dry);
	+ return $this->adjustSchemata(false);
	}
	}

	}
	diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php
	index 3203998a40..cbb88ce818 100644
	--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php
	+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php
	@@ -1,683 +1,902 @@
	<?php

	abstract class PhabricatorStorageManagementWorkflow
	extends PhabricatorManagementWorkflow {

	- private $patches;
	private $api;
	+ private $dryRun;
	+ private $force;
	+ private $patches;

	- public function setPatches(array $patches) {
	- assert_instances_of($patches, 'PhabricatorStoragePatch');
	- $this->patches = $patches;
	+ final public function getAPI() {
	+ return $this->api;
	+ }
	+
	+ final public function setAPI(PhabricatorStorageManagementAPI $api) {
	+ $this->api = $api;
	+ return $this;
	+ }
	+
	+ final protected function isDryRun() {
	+ return $this->dryRun;
	+ }
	+
	+ final protected function setDryRun($dry_run) {
	+ $this->dryRun = $dry_run;
	+ return $this;
	+ }
	+
	+ final protected function isForce() {
	+ return $this->force;
	+ }
	+
	+ final protected function setForce($force) {
	+ $this->force = $force;
	return $this;
	}

	public function getPatches() {
	return $this->patches;
	}

	- final public function setAPI(PhabricatorStorageManagementAPI $api) {
	- $this->api = $api;
	+ public function setPatches(array $patches) {
	+ assert_instances_of($patches, 'PhabricatorStoragePatch');
	+ $this->patches = $patches;
	return $this;
	}

	- final public function getAPI() {
	- return $this->api;
	+
	+ public function execute(PhutilArgumentParser $args) {
	+ $this->setDryRun($args->getArg('dryrun'));
	+ $this->setForce($args->getArg('force'));
	+
	+ $this->didExecute($args);
	}

	+ public function didExecute(PhutilArgumentParser $args) {}
	+
	private function loadSchemata() {
	$query = id(new PhabricatorConfigSchemaQuery())
	->setAPI($this->getAPI());

	$actual = $query->loadActualSchema();
	$expect = $query->loadExpectedSchema();
	$comp = $query->buildComparisonSchema($expect, $actual);

	return array($comp, $expect, $actual);
	}

	- protected function adjustSchemata($force, $unsafe, $dry_run) {
	+ final protected function adjustSchemata($unsafe) {
	+ $lock = $this->lock();
	+
	+ try {
	+ $this->doAdjustSchemata($unsafe);
	+ } catch (Exception $ex) {
	+ $lock->unlock();
	+ throw $ex;
	+ }
	+
	+ $lock->unlock();
	+ }
	+
	+ final private function doAdjustSchemata($unsafe) {
	$console = PhutilConsole::getConsole();

	$console->writeOut(
	"%s\n",
	pht('Verifying database schemata...'));

	list($adjustments, $errors) = $this->findAdjustments();
	$api = $this->getAPI();

	if (!$adjustments) {
	$console->writeOut(
	"%s\n",
	pht('Found no adjustments for schemata.'));

	return $this->printErrors($errors, 0);
	}

	- if (!$force && !$api->isCharacterSetAvailable('utf8mb4')) {
	+ if (!$this->force && !$api->isCharacterSetAvailable('utf8mb4')) {
	$message = pht(
	"You have an old version of MySQL (older than 5.5) which does not ".
	"support the utf8mb4 character set. We strongly recomend upgrading to ".
	"5.5 or newer.\n\n".
	"If you apply adjustments now and later update MySQL to 5.5 or newer, ".
	"you'll need to apply adjustments again (and they will take a long ".
	"time).\n\n".
	"You can exit this workflow, update MySQL now, and then run this ".
	"workflow again. This is recommended, but may cause a lot of downtime ".
	"right now.\n\n".
	"You can exit this workflow, continue using Phabricator without ".
	"applying adjustments, update MySQL at a later date, and then run ".
	"this workflow again. This is also a good approach, and will let you ".
	"delay downtime until later.\n\n".
	"You can proceed with this workflow, and then optionally update ".
	"MySQL at a later date. After you do, you'll need to apply ".
	"adjustments again.\n\n".
	"For more information, see \"Managing Storage Adjustments\" in ".
	"the documentation.");

	$console->writeOut(
	"\n<bg:yellow> %s </bg>\n\n%s\n",
	pht('OLD MySQL VERSION'),
	phutil_console_wrap($message));

	$prompt = pht('Continue with old MySQL version?');
	if (!phutil_console_confirm($prompt, $default_no = true)) {
	return;
	}
	}

	$table = id(new PhutilConsoleTable())
	->addColumn('database', array('title' => pht('Database')))
	->addColumn('table', array('title' => pht('Table')))
	->addColumn('name', array('title' => pht('Name')))
	->addColumn('info', array('title' => pht('Issues')));

	foreach ($adjustments as $adjust) {
	$info = array();
	foreach ($adjust['issues'] as $issue) {
	$info[] = PhabricatorConfigStorageSchema::getIssueName($issue);
	}

	$table->addRow(array(
	'database' => $adjust['database'],
	'table' => idx($adjust, 'table'),
	'name' => idx($adjust, 'name'),
	'info' => implode(', ', $info),
	));
	}

	$console->writeOut("\n\n");

	$table->draw();

	- if ($dry_run) {
	+ if ($this->dryRun) {
	$console->writeOut(
	"%s\n",
	pht('DRYRUN: Would apply adjustments.'));
	return 0;
	- } else if (!$force) {
	+ } else if (!$this->force) {
	$console->writeOut(
	"\n%s\n",
	pht(
	"Found %s issues(s) with schemata, detailed above.\n\n".
	"You can review issues in more detail from the web interface, ".
	"in Config > Database Status. To better understand the adjustment ".
	"workflow, see \"Managing Storage Adjustments\" in the ".
	"documentation.\n\n".
	"MySQL needs to copy table data to make some adjustments, so these ".
	"migrations may take some time.",
	phutil_count($adjustments)));

	$prompt = pht('Fix these schema issues?');
	if (!phutil_console_confirm($prompt, $default_no = true)) {
	return 1;
	}
	}

	$console->writeOut(
	"%s\n",
	pht('Fixing schema issues...'));

	$conn = $api->getConn(null);

	if ($unsafe) {
	queryfx($conn, 'SET SESSION sql_mode = %s', '');
	} else {
	queryfx($conn, 'SET SESSION sql_mode = %s', 'STRICT_ALL_TABLES');
	}

	$failed = array();

	// We make changes in several phases.
	$phases = array(
	// Drop surplus autoincrements. This allows us to drop primary keys on
	// autoincrement columns.
	'drop_auto',

	// Drop all keys we're going to adjust. This prevents them from
	// interfering with column changes.
	'drop_keys',

	// Apply all database, table, and column changes.
	'main',

	// Restore adjusted keys.
	'add_keys',

	// Add missing autoincrements.
	'add_auto',
	);

	$bar = id(new PhutilConsoleProgressBar())
	->setTotal(count($adjustments) * count($phases));

	foreach ($phases as $phase) {
	foreach ($adjustments as $adjust) {
	try {
	switch ($adjust['kind']) {
	case 'database':
	if ($phase == 'main') {
	queryfx(
	$conn,
	'ALTER DATABASE %T CHARACTER SET = %s COLLATE = %s',
	$adjust['database'],
	$adjust['charset'],
	$adjust['collation']);
	}
	break;
	case 'table':
	if ($phase == 'main') {
	queryfx(
	$conn,
	'ALTER TABLE %T.%T COLLATE = %s',
	$adjust['database'],
	$adjust['table'],
	$adjust['collation']);
	}
	break;
	case 'column':
	$apply = false;
	$auto = false;
	$new_auto = idx($adjust, 'auto');
	if ($phase == 'drop_auto') {
	if ($new_auto === false) {
	$apply = true;
	$auto = false;
	}
	} else if ($phase == 'main') {
	$apply = true;
	if ($new_auto === false) {
	$auto = false;
	} else {
	$auto = $adjust['is_auto'];
	}
	} else if ($phase == 'add_auto') {
	if ($new_auto === true) {
	$apply = true;
	$auto = true;
	}
	}

	if ($apply) {
	$parts = array();

	if ($auto) {
	$parts[] = qsprintf(
	$conn,
	'AUTO_INCREMENT');
	}

	if ($adjust['charset']) {
	$parts[] = qsprintf(
	$conn,
	'CHARACTER SET %Q COLLATE %Q',
	$adjust['charset'],
	$adjust['collation']);
	}

	queryfx(
	$conn,
	'ALTER TABLE %T.%T MODIFY %T %Q %Q %Q',
	$adjust['database'],
	$adjust['table'],
	$adjust['name'],
	$adjust['type'],
	implode(' ', $parts),
	$adjust['nullable'] ? 'NULL' : 'NOT NULL');
	}
	break;
	case 'key':
	if (($phase == 'drop_keys') && $adjust['exists']) {
	if ($adjust['name'] == 'PRIMARY') {
	$key_name = 'PRIMARY KEY';
	} else {
	$key_name = qsprintf($conn, 'KEY %T', $adjust['name']);
	}

	queryfx(
	$conn,
	'ALTER TABLE %T.%T DROP %Q',
	$adjust['database'],
	$adjust['table'],
	$key_name);
	}

	if (($phase == 'add_keys') && $adjust['keep']) {
	// Different keys need different creation syntax. Notable
	// special cases are primary keys and fulltext keys.
	if ($adjust['name'] == 'PRIMARY') {
	$key_name = 'PRIMARY KEY';
	} else if ($adjust['indexType'] == 'FULLTEXT') {
	$key_name = qsprintf($conn, 'FULLTEXT %T', $adjust['name']);
	} else {
	if ($adjust['unique']) {
	$key_name = qsprintf(
	$conn,
	'UNIQUE KEY %T',
	$adjust['name']);
	} else {
	$key_name = qsprintf(
	$conn,
	'/* NONUNIQUE */ KEY %T',
	$adjust['name']);
	}
	}

	queryfx(
	$conn,
	'ALTER TABLE %T.%T ADD %Q (%Q)',
	$adjust['database'],
	$adjust['table'],
	$key_name,
	implode(', ', $adjust['columns']));
	}
	break;
	default:
	throw new Exception(
	pht('Unknown schema adjustment kind "%s"!', $adjust['kind']));
	}
	} catch (AphrontQueryException $ex) {
	$failed[] = array($adjust, $ex);
	}
	$bar->update(1);
	}
	}
	$bar->done();

	if (!$failed) {
	$console->writeOut(
	"%s\n",
	pht('Completed fixing all schema issues.'));

	$err = 0;
	} else {
	$table = id(new PhutilConsoleTable())
	->addColumn('target', array('title' => pht('Target')))
	->addColumn('error', array('title' => pht('Error')));

	foreach ($failed as $failure) {
	list($adjust, $ex) = $failure;

	$pieces = array_select_keys(
	$adjust,
	array('database', 'table', 'name'));
	$pieces = array_filter($pieces);
	$target = implode('.', $pieces);

	$table->addRow(
	array(
	'target' => $target,
	'error' => $ex->getMessage(),
	));
	}

	$console->writeOut("\n");
	$table->draw();
	$console->writeOut(
	"\n%s\n",
	pht('Failed to make some schema adjustments, detailed above.'));
	$console->writeOut(
	"%s\n",
	pht(
	'For help troubleshooting adjustments, see "Managing Storage '.
	'Adjustments" in the documentation.'));

	$err = 1;
	}

	return $this->printErrors($errors, $err);
	}

	private function findAdjustments() {
	list($comp, $expect, $actual) = $this->loadSchemata();

	$issue_charset = PhabricatorConfigStorageSchema::ISSUE_CHARSET;
	$issue_collation = PhabricatorConfigStorageSchema::ISSUE_COLLATION;
	$issue_columntype = PhabricatorConfigStorageSchema::ISSUE_COLUMNTYPE;
	$issue_surpluskey = PhabricatorConfigStorageSchema::ISSUE_SURPLUSKEY;
	$issue_missingkey = PhabricatorConfigStorageSchema::ISSUE_MISSINGKEY;
	$issue_columns = PhabricatorConfigStorageSchema::ISSUE_KEYCOLUMNS;
	$issue_unique = PhabricatorConfigStorageSchema::ISSUE_UNIQUE;
	$issue_longkey = PhabricatorConfigStorageSchema::ISSUE_LONGKEY;
	$issue_auto = PhabricatorConfigStorageSchema::ISSUE_AUTOINCREMENT;

	$adjustments = array();
	$errors = array();
	foreach ($comp->getDatabases() as $database_name => $database) {
	foreach ($this->findErrors($database) as $issue) {
	$errors[] = array(
	'database' => $database_name,
	'issue' => $issue,
	);
	}

	$expect_database = $expect->getDatabase($database_name);
	$actual_database = $actual->getDatabase($database_name);

	if (!$expect_database \|\| !$actual_database) {
	// If there's a real issue here, skip this stuff.
	continue;
	}

	$issues = array();
	if ($database->hasIssue($issue_charset)) {
	$issues[] = $issue_charset;
	}
	if ($database->hasIssue($issue_collation)) {
	$issues[] = $issue_collation;
	}

	if ($issues) {
	$adjustments[] = array(
	'kind' => 'database',
	'database' => $database_name,
	'issues' => $issues,
	'charset' => $expect_database->getCharacterSet(),
	'collation' => $expect_database->getCollation(),
	);
	}

	foreach ($database->getTables() as $table_name => $table) {
	foreach ($this->findErrors($table) as $issue) {
	$errors[] = array(
	'database' => $database_name,
	'table' => $table_name,
	'issue' => $issue,
	);
	}

	$expect_table = $expect_database->getTable($table_name);
	$actual_table = $actual_database->getTable($table_name);

	if (!$expect_table \|\| !$actual_table) {
	continue;
	}

	$issues = array();
	if ($table->hasIssue($issue_collation)) {
	$issues[] = $issue_collation;
	}

	if ($issues) {
	$adjustments[] = array(
	'kind' => 'table',
	'database' => $database_name,
	'table' => $table_name,
	'issues' => $issues,
	'collation' => $expect_table->getCollation(),
	);
	}

	foreach ($table->getColumns() as $column_name => $column) {
	foreach ($this->findErrors($column) as $issue) {
	$errors[] = array(
	'database' => $database_name,
	'table' => $table_name,
	'name' => $column_name,
	'issue' => $issue,
	);
	}

	$expect_column = $expect_table->getColumn($column_name);
	$actual_column = $actual_table->getColumn($column_name);

	if (!$expect_column \|\| !$actual_column) {
	continue;
	}

	$issues = array();
	if ($column->hasIssue($issue_collation)) {
	$issues[] = $issue_collation;
	}
	if ($column->hasIssue($issue_charset)) {
	$issues[] = $issue_charset;
	}
	if ($column->hasIssue($issue_columntype)) {
	$issues[] = $issue_columntype;
	}
	if ($column->hasIssue($issue_auto)) {
	$issues[] = $issue_auto;
	}

	if ($issues) {
	if ($expect_column->getCharacterSet() === null) {
	// For non-text columns, we won't be specifying a collation or
	// character set.
	$charset = null;
	$collation = null;
	} else {
	$charset = $expect_column->getCharacterSet();
	$collation = $expect_column->getCollation();
	}

	$adjustment = array(
	'kind' => 'column',
	'database' => $database_name,
	'table' => $table_name,
	'name' => $column_name,
	'issues' => $issues,
	'collation' => $collation,
	'charset' => $charset,
	'type' => $expect_column->getColumnType(),

	// NOTE: We don't adjust column nullability because it is
	// dangerous, so always use the current nullability.
	'nullable' => $actual_column->getNullable(),

	// NOTE: This always stores the current value, because we have
	// to make these updates separately.
	'is_auto' => $actual_column->getAutoIncrement(),
	);

	if ($column->hasIssue($issue_auto)) {
	$adjustment['auto'] = $expect_column->getAutoIncrement();
	}

	$adjustments[] = $adjustment;
	}
	}

	foreach ($table->getKeys() as $key_name => $key) {
	foreach ($this->findErrors($key) as $issue) {
	$errors[] = array(
	'database' => $database_name,
	'table' => $table_name,
	'name' => $key_name,
	'issue' => $issue,
	);
	}

	$expect_key = $expect_table->getKey($key_name);
	$actual_key = $actual_table->getKey($key_name);

	$issues = array();
	$keep_key = true;
	if ($key->hasIssue($issue_surpluskey)) {
	$issues[] = $issue_surpluskey;
	$keep_key = false;
	}

	if ($key->hasIssue($issue_missingkey)) {
	$issues[] = $issue_missingkey;
	}

	if ($key->hasIssue($issue_columns)) {
	$issues[] = $issue_columns;
	}

	if ($key->hasIssue($issue_unique)) {
	$issues[] = $issue_unique;
	}

	// NOTE: We can't really fix this, per se, but we may need to remove
	// the key to change the column type. In the best case, the new
	// column type won't be overlong and recreating the key really will
	// fix the issue. In the worst case, we get the right column type and
	// lose the key, which is still better than retaining the key having
	// the wrong column type.
	if ($key->hasIssue($issue_longkey)) {
	$issues[] = $issue_longkey;
	}

	if ($issues) {
	$adjustment = array(
	'kind' => 'key',
	'database' => $database_name,
	'table' => $table_name,
	'name' => $key_name,
	'issues' => $issues,
	'exists' => (bool)$actual_key,
	'keep' => $keep_key,
	);

	if ($keep_key) {
	$adjustment += array(
	'columns' => $expect_key->getColumnNames(),
	'unique' => $expect_key->getUnique(),
	'indexType' => $expect_key->getIndexType(),
	);
	}

	$adjustments[] = $adjustment;
	}
	}
	}
	}

	return array($adjustments, $errors);
	}

	private function findErrors(PhabricatorConfigStorageSchema $schema) {
	$result = array();
	foreach ($schema->getLocalIssues() as $issue) {
	$status = PhabricatorConfigStorageSchema::getIssueStatus($issue);
	if ($status == PhabricatorConfigStorageSchema::STATUS_FAIL) {
	$result[] = $issue;
	}
	}
	return $result;
	}

	private function printErrors(array $errors, $default_return) {
	if (!$errors) {
	return $default_return;
	}

	$console = PhutilConsole::getConsole();

	$table = id(new PhutilConsoleTable())
	->addColumn('target', array('title' => pht('Target')))
	->addColumn('error', array('title' => pht('Error')));

	$any_surplus = false;
	$all_surplus = true;
	foreach ($errors as $error) {
	$pieces = array_select_keys(
	$error,
	array('database', 'table', 'name'));
	$pieces = array_filter($pieces);
	$target = implode('.', $pieces);

	$name = PhabricatorConfigStorageSchema::getIssueName($error['issue']);

	if ($error['issue'] === PhabricatorConfigStorageSchema::ISSUE_SURPLUS) {
	$any_surplus = true;
	} else {
	$all_surplus = false;
	}

	$table->addRow(
	array(
	'target' => $target,
	'error' => $name,
	));
	}

	$console->writeOut("\n");
	$table->draw();
	$console->writeOut("\n");


	$message = array();
	if ($all_surplus) {
	$message[] = pht(
	'You have surplus schemata (extra tables or columns which Phabricator '.
	'does not expect). For information on resolving these '.
	'issues, see the "Surplus Schemata" section in the "Managing Storage '.
	'Adjustments" article in the documentation.');
	} else {
	$message[] = pht(
	'The schemata have errors (detailed above) which the adjustment '.
	'workflow can not fix.');

	if ($any_surplus) {
	$message[] = pht(
	'Some of these errors are caused by surplus schemata (extra '.
	'tables or columns which Phabricator does not expect). These are '.
	'not serious. For information on resolving these issues, see the '.
	'"Surplus Schemata" section in the "Managing Storage Adjustments" '.
	'article in the documentation.');
	}

	$message[] = pht(
	'If you are not developing Phabricator itself, report this issue to '.
	'the upstream.');

	$message[] = pht(
	'If you are developing Phabricator, these errors usually indicate '.
	'that your schema specifications do not agree with the schemata your '.
	'code actually builds.');
	}
	$message = implode("\n\n", $message);

	if ($all_surplus) {
	$console->writeOut(
	"<bg:yellow> %s </bg>\n\n%s\n",
	pht('SURPLUS SCHEMATA'),
	phutil_console_wrap($message));
	} else {
	$console->writeOut(
	"<bg:red> %s </bg>\n\n%s\n",
	pht('SCHEMATA ERRORS'),
	phutil_console_wrap($message));
	}

	return 2;
	}

	+ final protected function upgradeSchemata(
	+ $apply_only = null,
	+ $no_quickstart = false,
	+ $init_only = false) {
	+
	+ $lock = $this->lock();
	+
	+ try {
	+ $this->doUpgradeSchemata($apply_only, $no_quickstart, $init_only);
	+ } catch (Exception $ex) {
	+ $lock->unlock();
	+ throw $ex;
	+ }
	+
	+ $lock->unlock();
	+ }
	+
	+ final private function doUpgradeSchemata(
	+ $apply_only,
	+ $no_quickstart,
	+ $init_only) {
	+
	+ $api = $this->getAPI();
	+
	+ $applied = $this->getApi()->getAppliedPatches();
	+ if ($applied === null) {
	+ if ($this->dryRun) {
	+ echo pht(
	+ "DRYRUN: Patch metadata storage doesn't exist yet, ".
	+ "it would be created.\n");
	+ return 0;
	+ }
	+
	+ if ($apply_only) {
	+ throw new PhutilArgumentUsageException(
	+ pht(
	+ 'Storage has not been initialized yet, you must initialize '.
	+ 'storage before selectively applying patches.'));
	+ return 1;
	+ }
	+
	+ $legacy = $api->getLegacyPatches($this->patches);
	+ if ($legacy \|\| $no_quickstart \|\| $init_only) {
	+
	+ // If we have legacy patches, we can't quickstart.
	+
	+ $api->createDatabase('meta_data');
	+ $api->createTable(
	+ 'meta_data',
	+ 'patch_status',
	+ array(
	+ 'patch VARCHAR(255) NOT NULL PRIMARY KEY COLLATE utf8_general_ci',
	+ 'applied INT UNSIGNED NOT NULL',
	+ ));
	+
	+ foreach ($legacy as $patch) {
	+ $api->markPatchApplied($patch);
	+ }
	+ } else {
	+ echo pht('Loading quickstart template...')."\n";
	+ $root = dirname(phutil_get_library_root('phabricator'));
	+ $sql = $root.'/resources/sql/quickstart.sql';
	+ $api->applyPatchSQL($sql);
	+ }
	+ }
	+
	+ if ($init_only) {
	+ echo pht('Storage initialized.')."\n";
	+ return 0;
	+ }
	+
	+ $applied = $api->getAppliedPatches();
	+ $applied = array_fuse($applied);
	+
	+ $skip_mark = false;
	+ if ($apply_only) {
	+ if (isset($applied[$apply_only])) {
	+
	+ unset($applied[$apply_only]);
	+ $skip_mark = true;
	+
	+ if (!$this->force && !$this->dryRun) {
	+ echo phutil_console_wrap(
	+ pht(
	+ "Patch '%s' has already been applied. Are you sure you want ".
	+ "to apply it again? This may put your storage in a state ".
	+ "that the upgrade scripts can not automatically manage.",
	+ $apply_only));
	+ if (!phutil_console_confirm(pht('Apply patch again?'))) {
	+ echo pht('Cancelled.')."\n";
	+ return 1;
	+ }
	+ }
	+ }
	+ }
	+
	+ while (true) {
	+ $applied_something = false;
	+ foreach ($this->patches as $key => $patch) {
	+ if (isset($applied[$key])) {
	+ unset($this->patches[$key]);
	+ continue;
	+ }
	+
	+ if ($apply_only && $apply_only != $key) {
	+ unset($this->patches[$key]);
	+ continue;
	+ }
	+
	+ $can_apply = true;
	+ foreach ($patch->getAfter() as $after) {
	+ if (empty($applied[$after])) {
	+ if ($apply_only) {
	+ echo pht(
	+ "Unable to apply patch '%s' because it depends ".
	+ "on patch '%s', which has not been applied.\n",
	+ $apply_only,
	+ $after);
	+ return 1;
	+ }
	+ $can_apply = false;
	+ break;
	+ }
	+ }
	+
	+ if (!$can_apply) {
	+ continue;
	+ }
	+
	+ $applied_something = true;
	+
	+ if ($this->dryRun) {
	+ echo pht("DRYRUN: Would apply patch '%s'.", $key)."\n";
	+ } else {
	+ echo pht("Applying patch '%s'...", $key)."\n";
	+
	+ $t_begin = microtime(true);
	+ $api->applyPatch($patch);
	+ $t_end = microtime(true);
	+
	+ if (!$skip_mark) {
	+ $api->markPatchApplied($key, ($t_end - $t_begin));
	+ }
	+ }
	+
	+ unset($this->patches[$key]);
	+ $applied[$key] = true;
	+ }
	+
	+ if (!$applied_something) {
	+ if (count($this->patches)) {
	+ throw new Exception(
	+ pht(
	+ 'Some patches could not be applied: %s',
	+ implode(', ', array_keys($this->patches))));
	+ } else if (!$this->dryRun && !$apply_only) {
	+ echo pht(
	+ "Storage is up to date. Use '%s' for details.",
	+ 'storage status')."\n";
	+ }
	+ break;
	+ }
	+ }
	+ }
	+
	final protected function getBareHostAndPort($host) {
	// Split out port information, since the command-line client requires a
	// separate flag for the port.
	$uri = new PhutilURI('mysql://'.$host);
	if ($uri->getPort()) {
	$port = $uri->getPort();
	$bare_hostname = $uri->getDomain();
	} else {
	$port = null;
	$bare_hostname = $host;
	}

	return array($bare_hostname, $port);
	}

	+ /**
	+ * Acquires a @{class:PhabricatorGlobalLock}.
	+ *
	+ * @return PhabricatorGlobalLock
	+ */
	+ final protected function lock() {
	+ return PhabricatorGlobalLock::newLock(__CLASS__)
	+ ->useSpecificConnection($this->getApi()->getConn(null))
	+ ->lock();
	+ }
	+
	}
	diff --git a/src/infrastructure/util/PhabricatorGlobalLock.php b/src/infrastructure/util/PhabricatorGlobalLock.php
	index 41558531d7..394f57d9a9 100644
	--- a/src/infrastructure/util/PhabricatorGlobalLock.php
	+++ b/src/infrastructure/util/PhabricatorGlobalLock.php
	@@ -1,122 +1,137 @@
	<?php

	/**
	* Global, MySQL-backed lock. This is a high-reliability, low-performance
	* global lock.
	*
	* The lock is maintained by using GET_LOCK() in MySQL, and automatically
	* released when the connection terminates. Thus, this lock can safely be used
	* to control access to shared resources without implementing any sort of
	* timeout or override logic: the lock can't normally be stuck in a locked state
	* with no process actually holding the lock.
	*
	* However, acquiring the lock is moderately expensive (several network
	* roundtrips). This makes it unsuitable for tasks where lock performance is
	* important.
	*
	* $lock = PhabricatorGlobalLock::newLock('example');
	* $lock->lock();
	* do_contentious_things();
	* $lock->unlock();
	*
	* NOTE: This lock is not completely global; it is namespaced to the active
	* storage namespace so that unit tests running in separate table namespaces
	* are isolated from one another.
	*
	* @task construct Constructing Locks
	* @task impl Implementation
	*/
	final class PhabricatorGlobalLock extends PhutilLock {

	private $conn;

	private static $pool = array();


	/* -( Constructing Locks )------------------------------------------------- */


	public static function newLock($name) {
	$namespace = PhabricatorLiskDAO::getStorageNamespace();
	$namespace = PhabricatorHash::digestToLength($namespace, 20);

	$full_name = 'ph:'.$namespace.':'.$name;

	$length_limit = 64;
	if (strlen($full_name) > $length_limit) {
	throw new Exception(
	pht(
	'Lock name "%s" is too long (full lock name is "%s"). The '.
	'full lock name must not be longer than %s bytes.',
	$name,
	$full_name,
	new PhutilNumber($length_limit)));
	}

	$lock = self::getLock($full_name);
	if (!$lock) {
	$lock = new PhabricatorGlobalLock($full_name);
	self::registerLock($lock);
	}

	return $lock;
	}

	+ /**
	+ * Use a specific database connection for locking.
	+ *
	+ * By default, `PhabricatorGlobalLock` will lock on the "repository" database
	+ * (somewhat arbitrarily). In most cases this is fine, but this method can
	+ * be used to lock on a specific connection.
	+ *
	+ * @param AphrontDatabaseConnection
	+ * @return this
	+ */
	+ public function useSpecificConnection(AphrontDatabaseConnection $conn) {
	+ $this->conn = $conn;
	+ return $this;
	+ }
	+

	/* -( Implementation )----------------------------------------------------- */

	protected function doLock($wait) {
	$conn = $this->conn;

	if (!$conn) {
	// Try to reuse a connection from the connection pool.
	$conn = array_pop(self::$pool);
	}

	if (!$conn) {
	// NOTE: Using the 'repository' database somewhat arbitrarily, mostly
	// because the first client of locks is the repository daemons. We must
	// always use the same database for all locks, but don't access any
	// tables so we could use any valid database. We could build a
	// database-free connection instead, but that's kind of messy and we
	// might forget about it in the future if we vertically partition the
	// application.
	$dao = new PhabricatorRepository();

	// NOTE: Using "force_new" to make sure each lock is on its own
	// connection.
	$conn = $dao->establishConnection('w', $force_new = true);
	-
	- // NOTE: Since MySQL will disconnect us if we're idle for too long, we set
	- // the wait_timeout to an enormous value, to allow us to hold the
	- // connection open indefinitely (or, at least, for 24 days).
	- $max_allowed_timeout = 2147483;
	- queryfx($conn, 'SET wait_timeout = %d', $max_allowed_timeout);
	}

	+ // NOTE: Since MySQL will disconnect us if we're idle for too long, we set
	+ // the wait_timeout to an enormous value, to allow us to hold the
	+ // connection open indefinitely (or, at least, for 24 days).
	+ $max_allowed_timeout = 2147483;
	+ queryfx($conn, 'SET wait_timeout = %d', $max_allowed_timeout);
	+
	$result = queryfx_one(
	$conn,
	'SELECT GET_LOCK(%s, %f)',
	$this->getName(),
	$wait);

	$ok = head($result);
	if (!$ok) {
	throw new PhutilLockException($this->getName());
	}

	$this->conn = $conn;
	}

	protected function doUnlock() {
	queryfx(
	$this->conn,
	'SELECT RELEASE_LOCK(%s)',
	$this->getName());

	$this->conn->close();
	self::$pool[] = $this->conn;
	$this->conn = null;
	}

	}

File Metadata

Mime Type: text/x-diff
Expires: Wed, Apr 30, 3:40 PM (1 d, 10 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 108751
Default Alt Text: (73 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions