No OneTemporary
Actions

Size

71 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php b/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php
	index 97d6e1ac81..c9e78af0bc 100644
	--- a/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php
	+++ b/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php
	@@ -1,37 +1,40 @@
	<?php

	final class PhabricatorCacheSchemaSpec extends PhabricatorConfigSchemaSpec {

	public function buildSchemata() {
	$this->buildRawSchema(
	'cache',
	id(new PhabricatorKeyValueDatabaseCache())->getTableName(),
	array(
	'id' => 'auto64',
	'cacheKeyHash' => 'bytes12',
	'cacheKey' => 'text128',
	'cacheFormat' => 'text16',
	'cacheData' => 'bytes',
	'cacheCreated' => 'epoch',
	'cacheExpires' => 'epoch?',
	),
	array(
	'PRIMARY' => array(
	'columns' => array('id'),
	'unique' => true,
	),
	'key_cacheKeyHash' => array(
	'columns' => array('cacheKeyHash'),
	'unique' => true,
	),
	'key_cacheCreated' => array(
	'columns' => array('cacheCreated'),
	),
	'key_ttl' => array(
	'columns' => array('cacheExpires'),
	),
	+ ),
	+ array(
	+ 'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_CACHE,
	));

	}

	}
	diff --git a/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php b/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php
	index a67c57e6f9..760317ae80 100644
	--- a/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php
	+++ b/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php
	@@ -1,862 +1,865 @@
	<?php

	final class PhabricatorConfigDatabaseStatusController
	extends PhabricatorConfigDatabaseController {

	private $database;
	private $table;
	private $column;
	private $key;
	private $ref;

	public function handleRequest(AphrontRequest $request) {
	$viewer = $request->getViewer();
	$this->database = $request->getURIData('database');
	$this->table = $request->getURIData('table');
	$this->column = $request->getURIData('column');
	$this->key = $request->getURIData('key');
	$this->ref = $request->getURIData('ref');

	$query = new PhabricatorConfigSchemaQuery();

	$actual = $query->loadActualSchemata();
	$expect = $query->loadExpectedSchemata();
	$comp = $query->buildComparisonSchemata($expect, $actual);

	if ($this->ref !== null) {
	$server_actual = idx($actual, $this->ref);
	if (!$server_actual) {
	return new Aphront404Response();
	}

	$server_comparison = $comp[$this->ref];
	$server_expect = $expect[$this->ref];

	if ($this->column) {
	return $this->renderColumn(
	$server_comparison,
	$server_expect,
	$server_actual,
	$this->database,
	$this->table,
	$this->column);
	} else if ($this->key) {
	return $this->renderKey(
	$server_comparison,
	$server_expect,
	$server_actual,
	$this->database,
	$this->table,
	$this->key);
	} else if ($this->table) {
	return $this->renderTable(
	$server_comparison,
	$server_expect,
	$server_actual,
	$this->database,
	$this->table);
	} else if ($this->database) {
	return $this->renderDatabase(
	$server_comparison,
	$server_expect,
	$server_actual,
	$this->database);
	}
	}

	return $this->renderServers(
	$comp,
	$expect,
	$actual);
	}

	private function buildResponse($title, $body) {
	$nav = $this->buildSideNavView();
	$nav->selectFilter('database/');

	if (!$title) {
	$title = pht('Database Status');
	}

	$ref = $this->ref;
	$database = $this->database;
	$table = $this->table;
	$column = $this->column;
	$key = $this->key;

	$links = array();
	$links[] = array(
	pht('Database Status'),
	'database/',
	);

	if ($database) {
	$links[] = array(
	$database,
	"database/{$ref}/{$database}/",
	);
	}

	if ($table) {
	$links[] = array(
	$table,
	"database/{$ref}/{$database}/{$table}/",
	);
	}

	if ($column) {
	$links[] = array(
	$column,
	"database/{$ref}/{$database}/{$table}/col/{$column}/",
	);
	}

	if ($key) {
	$links[] = array(
	$key,
	"database/{$ref}/{$database}/{$table}/key/{$key}/",
	);
	}

	$crumbs = $this->buildApplicationCrumbs();
	$crumbs->setBorder(true);

	$last_key = last_key($links);
	foreach ($links as $link_key => $link) {
	list($name, $href) = $link;
	if ($link_key == $last_key) {
	$crumbs->addTextCrumb($name);
	} else {
	$crumbs->addTextCrumb($name, $this->getApplicationURI($href));
	}
	}

	$doc_link = PhabricatorEnv::getDoclink('Managing Storage Adjustments');
	$button = id(new PHUIButtonView())
	->setTag('a')
	->setIcon('fa-book')
	->setHref($doc_link)
	->setText(pht('Documentation'));

	$header = $this->buildHeaderView($title, $button);

	$content = id(new PHUITwoColumnView())
	->setHeader($header)
	->setNavigation($nav)
	->setFixed(true)
	->setMainColumn($body);

	return $this->newPage()
	->setTitle($title)
	->setCrumbs($crumbs)
	->appendChild($content);
	}


	private function renderServers(
	array $comp_servers,
	array $expect_servers,
	array $actual_servers) {

	$charset_issue = PhabricatorConfigStorageSchema::ISSUE_CHARSET;
	$collation_issue = PhabricatorConfigStorageSchema::ISSUE_COLLATION;

	$rows = array();
	foreach ($comp_servers as $ref_key => $comp) {
	$actual = $actual_servers[$ref_key];
	$expect = $expect_servers[$ref_key];
	foreach ($comp->getDatabases() as $database_name => $database) {
	$actual_database = $actual->getDatabase($database_name);
	if ($actual_database) {
	$charset = $actual_database->getCharacterSet();
	$collation = $actual_database->getCollation();
	} else {
	$charset = null;
	$collation = null;
	}

	$status = $database->getStatus();
	$issues = $database->getIssues();

	$uri = $this->getURI(
	array(
	'ref' => $ref_key,
	'database' => $database_name,
	));

	$rows[] = array(
	$this->renderIcon($status),
	$ref_key,
	phutil_tag(
	'a',
	array(
	'href' => $uri,
	),
	$database_name),
	$this->renderAttr($charset, $database->hasIssue($charset_issue)),
	$this->renderAttr($collation, $database->hasIssue($collation_issue)),
	);
	}
	}

	$table = id(new AphrontTableView($rows))
	->setHeaders(
	array(
	null,
	pht('Server'),
	pht('Database'),
	pht('Charset'),
	pht('Collation'),
	))
	->setColumnClasses(
	array(
	null,
	null,
	'wide pri',
	null,
	null,
	));

	$title = pht('Database Status');
	$properties = $this->buildProperties(
	array(
	),
	$comp->getIssues());
	$properties = $this->buildConfigBoxView(pht('Properties'), $properties);
	$table = $this->buildConfigBoxView(pht('Database'), $table);

	return $this->buildResponse($title, array($properties, $table));
	}

	private function renderDatabase(
	PhabricatorConfigServerSchema $comp,
	PhabricatorConfigServerSchema $expect,
	PhabricatorConfigServerSchema $actual,
	$database_name) {

	$collation_issue = PhabricatorConfigStorageSchema::ISSUE_COLLATION;

	$database = $comp->getDatabase($database_name);
	if (!$database) {
	return new Aphront404Response();
	}

	$rows = array();
	foreach ($database->getTables() as $table_name => $table) {
	$status = $table->getStatus();

	$uri = $this->getURI(
	array(
	'table' => $table_name,
	));

	$rows[] = array(
	$this->renderIcon($status),
	phutil_tag(
	'a',
	array(
	'href' => $uri,
	),
	$table_name),
	$this->renderAttr(
	$table->getCollation(),
	$table->hasIssue($collation_issue)),
	+ $table->getPersistenceTypeDisplayName(),
	);
	}

	$table = id(new AphrontTableView($rows))
	->setHeaders(
	array(
	null,
	pht('Table'),
	pht('Collation'),
	+ pht('Persistence'),
	))
	->setColumnClasses(
	array(
	null,
	'wide pri',
	null,
	+ null,
	));

	$title = $database_name;

	$actual_database = $actual->getDatabase($database_name);
	if ($actual_database) {
	$actual_charset = $actual_database->getCharacterSet();
	$actual_collation = $actual_database->getCollation();
	} else {
	$actual_charset = null;
	$actual_collation = null;
	}

	$expect_database = $expect->getDatabase($database_name);
	if ($expect_database) {
	$expect_charset = $expect_database->getCharacterSet();
	$expect_collation = $expect_database->getCollation();
	} else {
	$expect_charset = null;
	$expect_collation = null;
	}

	$properties = $this->buildProperties(
	array(
	array(
	pht('Server'),
	$this->ref,
	),
	array(
	pht('Character Set'),
	$actual_charset,
	),
	array(
	pht('Expected Character Set'),
	$expect_charset,
	),
	array(
	pht('Collation'),
	$actual_collation,
	),
	array(
	pht('Expected Collation'),
	$expect_collation,
	),
	),
	$database->getIssues());

	$properties = $this->buildConfigBoxView(pht('Properties'), $properties);
	$table = $this->buildConfigBoxView(pht('Database'), $table);

	return $this->buildResponse($title, array($properties, $table));
	}

	private function renderTable(
	PhabricatorConfigServerSchema $comp,
	PhabricatorConfigServerSchema $expect,
	PhabricatorConfigServerSchema $actual,
	$database_name,
	$table_name) {

	$type_issue = PhabricatorConfigStorageSchema::ISSUE_COLUMNTYPE;
	$charset_issue = PhabricatorConfigStorageSchema::ISSUE_CHARSET;
	$collation_issue = PhabricatorConfigStorageSchema::ISSUE_COLLATION;
	$nullable_issue = PhabricatorConfigStorageSchema::ISSUE_NULLABLE;
	$unique_issue = PhabricatorConfigStorageSchema::ISSUE_UNIQUE;
	$columns_issue = PhabricatorConfigStorageSchema::ISSUE_KEYCOLUMNS;
	$longkey_issue = PhabricatorConfigStorageSchema::ISSUE_LONGKEY;
	$auto_issue = PhabricatorConfigStorageSchema::ISSUE_AUTOINCREMENT;

	$database = $comp->getDatabase($database_name);
	if (!$database) {
	return new Aphront404Response();
	}

	$table = $database->getTable($table_name);
	if (!$table) {
	return new Aphront404Response();
	}

	$actual_database = $actual->getDatabase($database_name);
	$actual_table = null;
	if ($actual_database) {
	$actual_table = $actual_database->getTable($table_name);
	}

	$expect_database = $expect->getDatabase($database_name);
	$expect_table = null;
	if ($expect_database) {
	$expect_table = $expect_database->getTable($table_name);
	}

	$rows = array();
	foreach ($table->getColumns() as $column_name => $column) {
	$expect_column = null;
	if ($expect_table) {
	$expect_column = $expect_table->getColumn($column_name);
	}

	$status = $column->getStatus();

	$data_type = null;
	if ($expect_column) {
	$data_type = $expect_column->getDataType();
	}

	$uri = $this->getURI(
	array(
	'column' => $column_name,
	));

	$rows[] = array(
	$this->renderIcon($status),
	phutil_tag(
	'a',
	array(
	'href' => $uri,
	),
	$column_name),
	$data_type,
	$this->renderAttr(
	$column->getColumnType(),
	$column->hasIssue($type_issue)),
	$this->renderAttr(
	$this->renderBoolean($column->getNullable()),
	$column->hasIssue($nullable_issue)),
	$this->renderAttr(
	$this->renderBoolean($column->getAutoIncrement()),
	$column->hasIssue($auto_issue)),
	$this->renderAttr(
	$column->getCharacterSet(),
	$column->hasIssue($charset_issue)),
	$this->renderAttr(
	$column->getCollation(),
	$column->hasIssue($collation_issue)),
	);
	}

	$table_view = id(new AphrontTableView($rows))
	->setHeaders(
	array(
	null,
	pht('Column'),
	pht('Data Type'),
	pht('Column Type'),
	pht('Nullable'),
	pht('Autoincrement'),
	pht('Character Set'),
	pht('Collation'),
	))
	->setColumnClasses(
	array(
	null,
	'wide pri',
	null,
	null,
	null,
	null,
	null,
	));

	$key_rows = array();
	foreach ($table->getKeys() as $key_name => $key) {
	$expect_key = null;
	if ($expect_table) {
	$expect_key = $expect_table->getKey($key_name);
	}

	$status = $key->getStatus();

	$size = 0;
	foreach ($key->getColumnNames() as $column_spec) {
	list($column_name, $prefix) = $key->getKeyColumnAndPrefix($column_spec);
	$column = $table->getColumn($column_name);
	if (!$column) {
	$size = 0;
	break;
	}
	$size += $column->getKeyByteLength($prefix);
	}

	$size_formatted = null;
	if ($size) {
	$size_formatted = $this->renderAttr(
	$size,
	$key->hasIssue($longkey_issue));
	}

	$uri = $this->getURI(
	array(
	'key' => $key_name,
	));

	$key_rows[] = array(
	$this->renderIcon($status),
	phutil_tag(
	'a',
	array(
	'href' => $uri,
	),
	$key_name),
	$this->renderAttr(
	implode(', ', $key->getColumnNames()),
	$key->hasIssue($columns_issue)),
	$this->renderAttr(
	$this->renderBoolean($key->getUnique()),
	$key->hasIssue($unique_issue)),
	$size_formatted,
	);
	}

	$keys_view = id(new AphrontTableView($key_rows))
	->setHeaders(
	array(
	null,
	pht('Key'),
	pht('Columns'),
	pht('Unique'),
	pht('Size'),
	))
	->setColumnClasses(
	array(
	null,
	'wide pri',
	null,
	null,
	null,
	));

	$title = pht('%s.%s', $database_name, $table_name);

	if ($actual_table) {
	$actual_collation = $actual_table->getCollation();
	} else {
	$actual_collation = null;
	}

	if ($expect_table) {
	$expect_collation = $expect_table->getCollation();
	} else {
	$expect_collation = null;
	}

	$properties = $this->buildProperties(
	array(
	array(
	pht('Server'),
	$this->ref,
	),
	array(
	pht('Collation'),
	$actual_collation,
	),
	array(
	pht('Expected Collation'),
	$expect_collation,
	),
	),
	$table->getIssues());

	$box_header = pht('%s.%s', $database_name, $table_name);

	$properties = $this->buildConfigBoxView(pht('Properties'), $properties);
	$table = $this->buildConfigBoxView(pht('Database'), $table_view);
	$keys = $this->buildConfigBoxView(pht('Keys'), $keys_view);

	return $this->buildResponse(
	$title, array($properties, $table, $keys));
	}

	private function renderColumn(
	PhabricatorConfigServerSchema $comp,
	PhabricatorConfigServerSchema $expect,
	PhabricatorConfigServerSchema $actual,
	$database_name,
	$table_name,
	$column_name) {

	$database = $comp->getDatabase($database_name);
	if (!$database) {
	return new Aphront404Response();
	}

	$table = $database->getTable($table_name);
	if (!$table) {
	return new Aphront404Response();
	}

	$column = $table->getColumn($column_name);
	if (!$column) {
	return new Aphront404Response();
	}

	$actual_database = $actual->getDatabase($database_name);
	$actual_table = null;
	$actual_column = null;
	if ($actual_database) {
	$actual_table = $actual_database->getTable($table_name);
	if ($actual_table) {
	$actual_column = $actual_table->getColumn($column_name);
	}
	}

	$expect_database = $expect->getDatabase($database_name);
	$expect_table = null;
	$expect_column = null;
	if ($expect_database) {
	$expect_table = $expect_database->getTable($table_name);
	if ($expect_table) {
	$expect_column = $expect_table->getColumn($column_name);
	}
	}

	if ($actual_column) {
	$actual_coltype = $actual_column->getColumnType();
	$actual_charset = $actual_column->getCharacterSet();
	$actual_collation = $actual_column->getCollation();
	$actual_nullable = $actual_column->getNullable();
	$actual_auto = $actual_column->getAutoIncrement();
	} else {
	$actual_coltype = null;
	$actual_charset = null;
	$actual_collation = null;
	$actual_nullable = null;
	$actual_auto = null;
	}

	if ($expect_column) {
	$data_type = $expect_column->getDataType();
	$expect_coltype = $expect_column->getColumnType();
	$expect_charset = $expect_column->getCharacterSet();
	$expect_collation = $expect_column->getCollation();
	$expect_nullable = $expect_column->getNullable();
	$expect_auto = $expect_column->getAutoIncrement();
	} else {
	$data_type = null;
	$expect_coltype = null;
	$expect_charset = null;
	$expect_collation = null;
	$expect_nullable = null;
	$expect_auto = null;
	}


	$title = pht(
	'%s.%s.%s',
	$database_name,
	$table_name,
	$column_name);

	$properties = $this->buildProperties(
	array(
	array(
	pht('Server'),
	$this->ref,
	),
	array(
	pht('Data Type'),
	$data_type,
	),
	array(
	pht('Column Type'),
	$actual_coltype,
	),
	array(
	pht('Expected Column Type'),
	$expect_coltype,
	),
	array(
	pht('Character Set'),
	$actual_charset,
	),
	array(
	pht('Expected Character Set'),
	$expect_charset,
	),
	array(
	pht('Collation'),
	$actual_collation,
	),
	array(
	pht('Expected Collation'),
	$expect_collation,
	),
	array(
	pht('Nullable'),
	$this->renderBoolean($actual_nullable),
	),
	array(
	pht('Expected Nullable'),
	$this->renderBoolean($expect_nullable),
	),
	array(
	pht('Autoincrement'),
	$this->renderBoolean($actual_auto),
	),
	array(
	pht('Expected Autoincrement'),
	$this->renderBoolean($expect_auto),
	),
	),
	$column->getIssues());

	$properties = $this->buildConfigBoxView(pht('Properties'), $properties);

	return $this->buildResponse($title, $properties);
	}

	private function renderKey(
	PhabricatorConfigServerSchema $comp,
	PhabricatorConfigServerSchema $expect,
	PhabricatorConfigServerSchema $actual,
	$database_name,
	$table_name,
	$key_name) {

	$database = $comp->getDatabase($database_name);
	if (!$database) {
	return new Aphront404Response();
	}

	$table = $database->getTable($table_name);
	if (!$table) {
	return new Aphront404Response();
	}

	$key = $table->getKey($key_name);
	if (!$key) {
	return new Aphront404Response();
	}

	$actual_database = $actual->getDatabase($database_name);
	$actual_table = null;
	$actual_key = null;
	if ($actual_database) {
	$actual_table = $actual_database->getTable($table_name);
	if ($actual_table) {
	$actual_key = $actual_table->getKey($key_name);
	}
	}

	$expect_database = $expect->getDatabase($database_name);
	$expect_table = null;
	$expect_key = null;
	if ($expect_database) {
	$expect_table = $expect_database->getTable($table_name);
	if ($expect_table) {
	$expect_key = $expect_table->getKey($key_name);
	}
	}

	if ($actual_key) {
	$actual_columns = $actual_key->getColumnNames();
	$actual_unique = $actual_key->getUnique();
	} else {
	$actual_columns = array();
	$actual_unique = null;
	}

	if ($expect_key) {
	$expect_columns = $expect_key->getColumnNames();
	$expect_unique = $expect_key->getUnique();
	} else {
	$expect_columns = array();
	$expect_unique = null;
	}

	$title = pht(
	'%s.%s (%s)',
	$database_name,
	$table_name,
	$key_name);

	$properties = $this->buildProperties(
	array(
	array(
	pht('Server'),
	$this->ref,
	),
	array(
	pht('Unique'),
	$this->renderBoolean($actual_unique),
	),
	array(
	pht('Expected Unique'),
	$this->renderBoolean($expect_unique),
	),
	array(
	pht('Columns'),
	implode(', ', $actual_columns),
	),
	array(
	pht('Expected Columns'),
	implode(', ', $expect_columns),
	),
	),
	$key->getIssues());

	$properties = $this->buildConfigBoxView(pht('Properties'), $properties);

	return $this->buildResponse($title, $properties);
	}

	private function buildProperties(array $properties, array $issues) {
	$view = id(new PHUIPropertyListView())
	->setUser($this->getRequest()->getUser());

	foreach ($properties as $property) {
	list($key, $value) = $property;
	$view->addProperty($key, $value);
	}

	$status_view = new PHUIStatusListView();
	if (!$issues) {
	$status_view->addItem(
	id(new PHUIStatusItemView())
	->setIcon(PHUIStatusItemView::ICON_ACCEPT, 'green')
	->setTarget(pht('No Schema Issues')));
	} else {
	foreach ($issues as $issue) {
	$note = PhabricatorConfigStorageSchema::getIssueDescription($issue);

	$status = PhabricatorConfigStorageSchema::getIssueStatus($issue);
	switch ($status) {
	case PhabricatorConfigStorageSchema::STATUS_WARN:
	$icon = PHUIStatusItemView::ICON_WARNING;
	$color = 'yellow';
	break;
	case PhabricatorConfigStorageSchema::STATUS_FAIL:
	default:
	$icon = PHUIStatusItemView::ICON_REJECT;
	$color = 'red';
	break;
	}

	$item = id(new PHUIStatusItemView())
	->setTarget(PhabricatorConfigStorageSchema::getIssueName($issue))
	->setIcon($icon, $color)
	->setNote($note);

	$status_view->addItem($item);
	}
	}
	$view->addProperty(pht('Schema Status'), $status_view);

	return phutil_tag_div('config-page-property', $view);
	}

	private function getURI(array $properties) {
	$defaults = array(
	'ref' => $this->ref,
	'database' => $this->database,
	'table' => $this->table,
	'column' => $this->column,
	'key' => $this->key,
	);

	$properties = $properties + $defaults;
	$properties = array_select_keys($properties, array_keys($defaults));

	$parts = array();
	foreach ($properties as $key => $property) {
	if (!strlen($property)) {
	continue;
	}

	if ($key == 'column') {
	$parts[] = 'col';
	} else if ($key == 'key') {
	$parts[] = 'key';
	}

	$parts[] = $property;
	}

	if ($parts) {
	$parts = implode('/', $parts).'/';
	} else {
	$parts = null;
	}

	return $this->getApplicationURI('/database/'.$parts);
	}

	}
	diff --git a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php
	index 6feda7363c..2cb7763110 100644
	--- a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php
	+++ b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php
	@@ -1,378 +1,380 @@
	<?php

	final class PhabricatorConfigSchemaQuery extends Phobject {

	private $refs;
	private $apis;

	public function setRefs(array $refs) {
	$this->refs = $refs;
	return $this;
	}

	public function getRefs() {
	if (!$this->refs) {
	return PhabricatorDatabaseRef::getMasterDatabaseRefs();
	}
	return $this->refs;
	}

	public function setAPIs(array $apis) {
	$map = array();
	foreach ($apis as $api) {
	$map[$api->getRef()->getRefKey()] = $api;
	}
	$this->apis = $map;
	return $this;
	}

	private function getDatabaseNames(PhabricatorDatabaseRef $ref) {
	$api = $this->getAPI($ref);
	$patches = PhabricatorSQLPatchList::buildAllPatches();
	return $api->getDatabaseList(
	$patches,
	$only_living = true);
	}

	private function getAPI(PhabricatorDatabaseRef $ref) {
	$key = $ref->getRefKey();

	if (isset($this->apis[$key])) {
	return $this->apis[$key];
	}

	return id(new PhabricatorStorageManagementAPI())
	->setUser($ref->getUser())
	->setHost($ref->getHost())
	->setPort($ref->getPort())
	->setNamespace(PhabricatorLiskDAO::getDefaultStorageNamespace())
	->setPassword($ref->getPass());
	}

	public function loadActualSchemata() {
	$refs = $this->getRefs();

	$schemata = array();
	foreach ($refs as $ref) {
	$schema = $this->loadActualSchemaForServer($ref);
	$schemata[$schema->getRef()->getRefKey()] = $schema;
	}

	return $schemata;
	}

	private function loadActualSchemaForServer(PhabricatorDatabaseRef $ref) {
	$databases = $this->getDatabaseNames($ref);

	$conn = $ref->newManagementConnection();

	$tables = queryfx_all(
	$conn,
	'SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_COLLATION, ENGINE
	FROM INFORMATION_SCHEMA.TABLES
	WHERE TABLE_SCHEMA IN (%Ls)',
	$databases);

	$database_info = queryfx_all(
	$conn,
	'SELECT SCHEMA_NAME, DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME
	FROM INFORMATION_SCHEMA.SCHEMATA
	WHERE SCHEMA_NAME IN (%Ls)',
	$databases);
	$database_info = ipull($database_info, null, 'SCHEMA_NAME');

	// Find databases which exist, but which the user does not have permission
	// to see.
	$invisible_databases = array();
	foreach ($databases as $database_name) {
	if (isset($database_info[$database_name])) {
	continue;
	}

	try {
	queryfx($conn, 'SHOW TABLES IN %T', $database_name);
	} catch (AphrontAccessDeniedQueryException $ex) {
	// This database exists, the user just doesn't have permission to
	// see it.
	$invisible_databases[] = $database_name;
	} catch (AphrontSchemaQueryException $ex) {
	// This database is legitimately missing.
	}
	}

	$sql = array();
	foreach ($tables as $table) {
	$sql[] = qsprintf(
	$conn,
	'(TABLE_SCHEMA = %s AND TABLE_NAME = %s)',
	$table['TABLE_SCHEMA'],
	$table['TABLE_NAME']);
	}

	if ($sql) {
	$column_info = queryfx_all(
	$conn,
	'SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, CHARACTER_SET_NAME,
	COLLATION_NAME, COLUMN_TYPE, IS_NULLABLE, EXTRA
	FROM INFORMATION_SCHEMA.COLUMNS
	WHERE (%Q)',
	'('.implode(') OR (', $sql).')');
	$column_info = igroup($column_info, 'TABLE_SCHEMA');
	} else {
	$column_info = array();
	}

	// NOTE: Tables like KEY_COLUMN_USAGE and TABLE_CONSTRAINTS only contain
	// primary, unique, and foreign keys, so we can't use them here. We pull
	// indexes later on using SHOW INDEXES.

	$server_schema = id(new PhabricatorConfigServerSchema())
	->setRef($ref);

	$tables = igroup($tables, 'TABLE_SCHEMA');
	foreach ($tables as $database_name => $database_tables) {
	$info = $database_info[$database_name];

	$database_schema = id(new PhabricatorConfigDatabaseSchema())
	->setName($database_name)
	->setCharacterSet($info['DEFAULT_CHARACTER_SET_NAME'])
	->setCollation($info['DEFAULT_COLLATION_NAME']);

	$database_column_info = idx($column_info, $database_name, array());
	$database_column_info = igroup($database_column_info, 'TABLE_NAME');

	foreach ($database_tables as $table) {
	$table_name = $table['TABLE_NAME'];

	$table_schema = id(new PhabricatorConfigTableSchema())
	->setName($table_name)
	->setCollation($table['TABLE_COLLATION'])
	->setEngine($table['ENGINE']);

	$columns = idx($database_column_info, $table_name, array());
	foreach ($columns as $column) {
	if (strpos($column['EXTRA'], 'auto_increment') === false) {
	$auto_increment = false;
	} else {
	$auto_increment = true;
	}

	$column_schema = id(new PhabricatorConfigColumnSchema())
	->setName($column['COLUMN_NAME'])
	->setCharacterSet($column['CHARACTER_SET_NAME'])
	->setCollation($column['COLLATION_NAME'])
	->setColumnType($column['COLUMN_TYPE'])
	->setNullable($column['IS_NULLABLE'] == 'YES')
	->setAutoIncrement($auto_increment);

	$table_schema->addColumn($column_schema);
	}

	$key_parts = queryfx_all(
	$conn,
	'SHOW INDEXES FROM %T.%T',
	$database_name,
	$table_name);
	$keys = igroup($key_parts, 'Key_name');
	foreach ($keys as $key_name => $key_pieces) {
	$key_pieces = isort($key_pieces, 'Seq_in_index');
	$head = head($key_pieces);

	// This handles string indexes which index only a prefix of a field.
	$column_names = array();
	foreach ($key_pieces as $piece) {
	$name = $piece['Column_name'];
	if ($piece['Sub_part']) {
	$name = $name.'('.$piece['Sub_part'].')';
	}
	$column_names[] = $name;
	}

	$key_schema = id(new PhabricatorConfigKeySchema())
	->setName($key_name)
	->setColumnNames($column_names)
	->setUnique(!$head['Non_unique'])
	->setIndexType($head['Index_type']);

	$table_schema->addKey($key_schema);
	}

	$database_schema->addTable($table_schema);
	}

	$server_schema->addDatabase($database_schema);
	}

	foreach ($invisible_databases as $database_name) {
	$server_schema->addDatabase(
	id(new PhabricatorConfigDatabaseSchema())
	->setName($database_name)
	->setAccessDenied(true));
	}

	return $server_schema;
	}

	public function loadExpectedSchemata() {
	$refs = $this->getRefs();

	$schemata = array();
	foreach ($refs as $ref) {
	$schema = $this->loadExpectedSchemaForServer($ref);
	$schemata[$schema->getRef()->getRefKey()] = $schema;
	}

	return $schemata;
	}

	public function loadExpectedSchemaForServer(PhabricatorDatabaseRef $ref) {
	$databases = $this->getDatabaseNames($ref);
	$info = $this->getAPI($ref)->getCharsetInfo();

	$specs = id(new PhutilClassMapQuery())
	->setAncestorClass('PhabricatorConfigSchemaSpec')
	->execute();

	$server_schema = id(new PhabricatorConfigServerSchema())
	->setRef($ref);

	foreach ($specs as $spec) {
	$spec
	->setUTF8Charset(
	$info[PhabricatorStorageManagementAPI::CHARSET_DEFAULT])
	->setUTF8BinaryCollation(
	$info[PhabricatorStorageManagementAPI::COLLATE_TEXT])
	->setUTF8SortingCollation(
	$info[PhabricatorStorageManagementAPI::COLLATE_SORT])
	->setServer($server_schema)
	->buildSchemata($server_schema);
	}

	return $server_schema;
	}

	public function buildComparisonSchemata(
	array $expect_servers,
	array $actual_servers) {

	$schemata = array();
	foreach ($actual_servers as $key => $actual_server) {
	$schemata[$key] = $this->buildComparisonSchemaForServer(
	$expect_servers[$key],
	$actual_server);
	}

	return $schemata;
	}

	private function buildComparisonSchemaForServer(
	PhabricatorConfigServerSchema $expect,
	PhabricatorConfigServerSchema $actual) {

	$comp_server = $actual->newEmptyClone();

	$all_databases = $actual->getDatabases() + $expect->getDatabases();
	foreach ($all_databases as $database_name => $database_template) {
	$actual_database = $actual->getDatabase($database_name);
	$expect_database = $expect->getDatabase($database_name);

	$issues = $this->compareSchemata($expect_database, $actual_database);

	$comp_database = $database_template->newEmptyClone()
	->setIssues($issues);

	if (!$actual_database) {
	$actual_database = $expect_database->newEmptyClone();
	}

	if (!$expect_database) {
	$expect_database = $actual_database->newEmptyClone();
	}

	$all_tables =
	$actual_database->getTables() +
	$expect_database->getTables();
	foreach ($all_tables as $table_name => $table_template) {
	$actual_table = $actual_database->getTable($table_name);
	$expect_table = $expect_database->getTable($table_name);

	$issues = $this->compareSchemata($expect_table, $actual_table);

	$comp_table = $table_template->newEmptyClone()
	->setIssues($issues);

	if (!$actual_table) {
	$actual_table = $expect_table->newEmptyClone();
	}
	if (!$expect_table) {
	$expect_table = $actual_table->newEmptyClone();
	}

	$all_columns =
	$actual_table->getColumns() +
	$expect_table->getColumns();
	foreach ($all_columns as $column_name => $column_template) {
	$actual_column = $actual_table->getColumn($column_name);
	$expect_column = $expect_table->getColumn($column_name);

	$issues = $this->compareSchemata($expect_column, $actual_column);

	$comp_column = $column_template->newEmptyClone()
	->setIssues($issues);

	$comp_table->addColumn($comp_column);
	}

	$all_keys =
	$actual_table->getKeys() +
	$expect_table->getKeys();
	foreach ($all_keys as $key_name => $key_template) {
	$actual_key = $actual_table->getKey($key_name);
	$expect_key = $expect_table->getKey($key_name);

	$issues = $this->compareSchemata($expect_key, $actual_key);

	$comp_key = $key_template->newEmptyClone()
	->setIssues($issues);

	$comp_table->addKey($comp_key);
	}

	+ $comp_table->setPersistenceType($expect_table->getPersistenceType());
	+
	$comp_database->addTable($comp_table);
	}
	$comp_server->addDatabase($comp_database);
	}

	return $comp_server;
	}

	private function compareSchemata(
	PhabricatorConfigStorageSchema $expect = null,
	PhabricatorConfigStorageSchema $actual = null) {

	$expect_is_key = ($expect instanceof PhabricatorConfigKeySchema);
	$actual_is_key = ($actual instanceof PhabricatorConfigKeySchema);

	if ($expect_is_key \|\| $actual_is_key) {
	$missing_issue = PhabricatorConfigStorageSchema::ISSUE_MISSINGKEY;
	$surplus_issue = PhabricatorConfigStorageSchema::ISSUE_SURPLUSKEY;
	} else {
	$missing_issue = PhabricatorConfigStorageSchema::ISSUE_MISSING;
	$surplus_issue = PhabricatorConfigStorageSchema::ISSUE_SURPLUS;
	}

	if (!$expect && !$actual) {
	throw new Exception(pht('Can not compare two missing schemata!'));
	} else if ($expect && !$actual) {
	$issues = array($missing_issue);
	} else if ($actual && !$expect) {
	$issues = array($surplus_issue);
	} else {
	$issues = $actual->compareTo($expect);
	}

	return $issues;
	}


	}
	diff --git a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php
	index 971f5b828f..c451658682 100644
	--- a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php
	+++ b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php
	@@ -1,432 +1,453 @@
	<?php

	abstract class PhabricatorConfigSchemaSpec extends Phobject {

	private $server;
	private $utf8Charset;
	private $utf8BinaryCollation;
	private $utf8SortingCollation;

	const DATATYPE_UNKNOWN = '<unknown>';

	public function setUTF8SortingCollation($utf8_sorting_collation) {
	$this->utf8SortingCollation = $utf8_sorting_collation;
	return $this;
	}

	public function getUTF8SortingCollation() {
	return $this->utf8SortingCollation;
	}

	public function setUTF8BinaryCollation($utf8_binary_collation) {
	$this->utf8BinaryCollation = $utf8_binary_collation;
	return $this;
	}

	public function getUTF8BinaryCollation() {
	return $this->utf8BinaryCollation;
	}

	public function setUTF8Charset($utf8_charset) {
	$this->utf8Charset = $utf8_charset;
	return $this;
	}

	public function getUTF8Charset() {
	return $this->utf8Charset;
	}

	public function setServer(PhabricatorConfigServerSchema $server) {
	$this->server = $server;
	return $this;
	}

	public function getServer() {
	return $this->server;
	}

	abstract public function buildSchemata();

	protected function buildLiskObjectSchema(PhabricatorLiskDAO $object) {
	$this->buildRawSchema(
	$object->getApplicationName(),
	$object->getTableName(),
	$object->getSchemaColumns(),
	$object->getSchemaKeys());
	}

	protected function buildFerretIndexSchema(PhabricatorFerretEngine $engine) {
	+ $index_options = array(
	+ 'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_INDEX,
	+ );
	+
	$this->buildRawSchema(
	$engine->getApplicationName(),
	$engine->getDocumentTableName(),
	$engine->getDocumentSchemaColumns(),
	- $engine->getDocumentSchemaKeys());
	+ $engine->getDocumentSchemaKeys(),
	+ $index_options);

	$this->buildRawSchema(
	$engine->getApplicationName(),
	$engine->getFieldTableName(),
	$engine->getFieldSchemaColumns(),
	- $engine->getFieldSchemaKeys());
	+ $engine->getFieldSchemaKeys(),
	+ $index_options);

	$this->buildRawSchema(
	$engine->getApplicationName(),
	$engine->getNgramsTableName(),
	$engine->getNgramsSchemaColumns(),
	- $engine->getNgramsSchemaKeys());
	+ $engine->getNgramsSchemaKeys(),
	+ $index_options);

	$this->buildRawSchema(
	$engine->getApplicationName(),
	$engine->getCommonNgramsTableName(),
	$engine->getCommonNgramsSchemaColumns(),
	- $engine->getCommonNgramsSchemaKeys());
	+ $engine->getCommonNgramsSchemaKeys(),
	+ $index_options);
	}

	protected function buildRawSchema(
	$database_name,
	$table_name,
	array $columns,
	- array $keys) {
	+ array $keys,
	+ array $options = array()) {
	+
	+ PhutilTypeSpec::checkMap(
	+ $options,
	+ array(
	+ 'persistence' => 'optional string',
	+ ));
	+
	$database = $this->getDatabase($database_name);

	$table = $this->newTable($table_name);

	if (PhabricatorSearchDocument::isInnoDBFulltextEngineAvailable()) {
	$fulltext_engine = 'InnoDB';
	} else {
	$fulltext_engine = 'MyISAM';
	}

	foreach ($columns as $name => $type) {
	if ($type === null) {
	continue;
	}

	$details = $this->getDetailsForDataType($type);

	$column_type = $details['type'];
	$charset = $details['charset'];
	$collation = $details['collation'];
	$nullable = $details['nullable'];
	$auto = $details['auto'];

	$column = $this->newColumn($name)
	->setDataType($type)
	->setColumnType($column_type)
	->setCharacterSet($charset)
	->setCollation($collation)
	->setNullable($nullable)
	->setAutoIncrement($auto);

	// If this table has any FULLTEXT fields, we expect it to use the best
	// available FULLTEXT engine, which may not be InnoDB.
	switch ($type) {
	case 'fulltext':
	case 'fulltext?':
	$table->setEngine($fulltext_engine);
	break;
	}

	$table->addColumn($column);
	}

	foreach ($keys as $key_name => $key_spec) {
	if ($key_spec === null) {
	// This is a subclass removing a key which Lisk expects.
	continue;
	}

	$key = $this->newKey($key_name)
	->setColumnNames(idx($key_spec, 'columns', array()));

	$key->setUnique((bool)idx($key_spec, 'unique'));
	$key->setIndexType(idx($key_spec, 'type', 'BTREE'));

	$table->addKey($key);
	}

	+ $persistence_type = idx($options, 'persistence');
	+ if ($persistence_type !== null) {
	+ $table->setPersistenceType($persistence_type);
	+ }
	+
	$database->addTable($table);
	}

	protected function buildEdgeSchemata(PhabricatorLiskDAO $object) {
	$this->buildRawSchema(
	$object->getApplicationName(),
	PhabricatorEdgeConfig::TABLE_NAME_EDGE,
	array(
	'src' => 'phid',
	'type' => 'uint32',
	'dst' => 'phid',
	'dateCreated' => 'epoch',
	'seq' => 'uint32',
	'dataID' => 'id?',
	),
	array(
	'PRIMARY' => array(
	'columns' => array('src', 'type', 'dst'),
	'unique' => true,
	),
	'src' => array(
	'columns' => array('src', 'type', 'dateCreated', 'seq'),
	),
	'key_dst' => array(
	'columns' => array('dst', 'type', 'src'),
	'unique' => true,
	),
	));

	$this->buildRawSchema(
	$object->getApplicationName(),
	PhabricatorEdgeConfig::TABLE_NAME_EDGEDATA,
	array(
	'id' => 'auto',
	'data' => 'text',
	),
	array(
	'PRIMARY' => array(
	'columns' => array('id'),
	'unique' => true,
	),
	));
	}

	protected function getDatabase($name) {
	$server = $this->getServer();

	$database = $server->getDatabase($this->getNamespacedDatabase($name));
	if (!$database) {
	$database = $this->newDatabase($name);
	$server->addDatabase($database);
	}

	return $database;
	}

	protected function newDatabase($name) {
	return id(new PhabricatorConfigDatabaseSchema())
	->setName($this->getNamespacedDatabase($name))
	->setCharacterSet($this->getUTF8Charset())
	->setCollation($this->getUTF8BinaryCollation());
	}

	protected function getNamespacedDatabase($name) {
	$namespace = PhabricatorLiskDAO::getStorageNamespace();
	return $namespace.'_'.$name;
	}

	protected function newTable($name) {
	return id(new PhabricatorConfigTableSchema())
	->setName($name)
	->setCollation($this->getUTF8BinaryCollation())
	->setEngine('InnoDB');
	}

	protected function newColumn($name) {
	return id(new PhabricatorConfigColumnSchema())
	->setName($name);
	}

	protected function newKey($name) {
	return id(new PhabricatorConfigKeySchema())
	->setName($name);
	}

	public function getMaximumByteLengthForDataType($data_type) {
	$info = $this->getDetailsForDataType($data_type);
	return idx($info, 'bytes');
	}

	private function getDetailsForDataType($data_type) {
	$column_type = null;
	$charset = null;
	$collation = null;
	$auto = false;
	$bytes = null;

	// If the type ends with "?", make the column nullable.
	$nullable = false;
	if (preg_match('/\?$/', $data_type)) {
	$nullable = true;
	$data_type = substr($data_type, 0, -1);
	}

	// NOTE: MySQL allows fragments like "VARCHAR(32) CHARACTER SET binary",
	// but just interprets that to mean "VARBINARY(32)". The fragment is
	// totally disallowed in a MODIFY statement vs a CREATE TABLE statement.

	$is_binary = ($this->getUTF8Charset() == 'binary');
	$matches = null;
	$pattern = '/^(fulltext\|sort\|text\|char)(\d+)?\z/';
	if (preg_match($pattern, $data_type, $matches)) {

	// Limit the permitted column lengths under the theory that it would
	// be nice to eventually reduce this to a small set of standard lengths.

	static $valid_types = array(
	'text255' => true,
	'text160' => true,
	'text128' => true,
	'text64' => true,
	'text40' => true,
	'text32' => true,
	'text20' => true,
	'text16' => true,
	'text12' => true,
	'text8' => true,
	'text4' => true,
	'text' => true,
	'char3' => true,
	'sort255' => true,
	'sort128' => true,
	'sort64' => true,
	'sort32' => true,
	'sort' => true,
	'fulltext' => true,
	);

	if (empty($valid_types[$data_type])) {
	throw new Exception(pht('Unknown column type "%s"!', $data_type));
	}

	$type = $matches[1];
	$size = idx($matches, 2);

	if ($size) {
	$bytes = $size;
	}

	switch ($type) {
	case 'text':
	if ($is_binary) {
	if ($size) {
	$column_type = 'varbinary('.$size.')';
	} else {
	$column_type = 'longblob';
	}
	} else {
	if ($size) {
	$column_type = 'varchar('.$size.')';
	} else {
	$column_type = 'longtext';
	}
	}
	break;
	case 'sort':
	if ($size) {
	$column_type = 'varchar('.$size.')';
	} else {
	$column_type = 'longtext';
	}
	break;
	case 'fulltext';
	// MySQL (at least, under MyISAM) refuses to create a FULLTEXT index
	// on a LONGBLOB column. We'd also lose case insensitivity in search.
	// Force this column to utf8 collation. This will truncate results
	// with 4-byte UTF characters in their text, but work reasonably in
	// the majority of cases.
	$column_type = 'longtext';
	break;
	case 'char':
	$column_type = 'char('.$size.')';
	break;
	}

	switch ($type) {
	case 'text':
	case 'char':
	if ($is_binary) {
	// We leave collation and character set unspecified in order to
	// generate valid SQL.
	} else {
	$charset = $this->getUTF8Charset();
	$collation = $this->getUTF8BinaryCollation();
	}
	break;
	case 'sort':
	case 'fulltext':
	if ($is_binary) {
	$charset = 'utf8';
	} else {
	$charset = $this->getUTF8Charset();
	}
	$collation = $this->getUTF8SortingCollation();
	break;
	}
	} else {
	switch ($data_type) {
	case 'auto':
	$column_type = 'int(10) unsigned';
	$auto = true;
	break;
	case 'auto64':
	$column_type = 'bigint(20) unsigned';
	$auto = true;
	break;
	case 'id':
	case 'epoch':
	case 'uint32':
	$column_type = 'int(10) unsigned';
	break;
	case 'sint32':
	$column_type = 'int(10)';
	break;
	case 'id64':
	case 'uint64':
	$column_type = 'bigint(20) unsigned';
	break;
	case 'sint64':
	$column_type = 'bigint(20)';
	break;
	case 'phid':
	case 'policy';
	case 'hashpath64':
	case 'ipaddress':
	$column_type = 'varbinary(64)';
	break;
	case 'bytes64':
	$column_type = 'binary(64)';
	break;
	case 'bytes40':
	$column_type = 'binary(40)';
	break;
	case 'bytes32':
	$column_type = 'binary(32)';
	break;
	case 'bytes20':
	$column_type = 'binary(20)';
	break;
	case 'bytes12':
	$column_type = 'binary(12)';
	break;
	case 'bytes4':
	$column_type = 'binary(4)';
	break;
	case 'bytes':
	$column_type = 'longblob';
	break;
	case 'bool':
	$column_type = 'tinyint(1)';
	break;
	case 'double':
	$column_type = 'double';
	break;
	case 'date':
	$column_type = 'date';
	break;
	default:
	$column_type = self::DATATYPE_UNKNOWN;
	$charset = self::DATATYPE_UNKNOWN;
	$collation = self::DATATYPE_UNKNOWN;
	break;
	}
	}

	return array(
	'type' => $column_type,
	'charset' => $charset,
	'collation' => $collation,
	'nullable' => $nullable,
	'auto' => $auto,
	'bytes' => $bytes,
	);
	}

	}
	diff --git a/src/applications/config/schema/PhabricatorConfigTableSchema.php b/src/applications/config/schema/PhabricatorConfigTableSchema.php
	index 870c8ebddb..d6e5bfd9fa 100644
	--- a/src/applications/config/schema/PhabricatorConfigTableSchema.php
	+++ b/src/applications/config/schema/PhabricatorConfigTableSchema.php
	@@ -1,97 +1,123 @@
	<?php

	final class PhabricatorConfigTableSchema
	extends PhabricatorConfigStorageSchema {

	private $collation;
	private $engine;
	private $columns = array();
	private $keys = array();
	+ private $persistenceType = self::PERSISTENCE_DATA;
	+
	+ const PERSISTENCE_DATA = 'data';
	+ const PERSISTENCE_CACHE = 'cache';
	+ const PERSISTENCE_INDEX = 'index';

	public function addColumn(PhabricatorConfigColumnSchema $column) {
	$key = $column->getName();
	if (isset($this->columns[$key])) {
	throw new Exception(
	pht('Trying to add duplicate column "%s"!', $key));
	}
	$this->columns[$key] = $column;
	return $this;
	}

	public function addKey(PhabricatorConfigKeySchema $key) {
	$name = $key->getName();
	if (isset($this->keys[$name])) {
	throw new Exception(
	pht('Trying to add duplicate key "%s"!', $name));
	}
	$key->setTable($this);
	$this->keys[$name] = $key;
	return $this;
	}

	public function getColumns() {
	return $this->columns;
	}

	public function getColumn($key) {
	return idx($this->getColumns(), $key);
	}

	public function getKeys() {
	return $this->keys;
	}

	public function getKey($key) {
	return idx($this->getKeys(), $key);
	}

	+ public function setPersistenceType($persistence_type) {
	+ $this->persistenceType = $persistence_type;
	+ return $this;
	+ }
	+
	+ public function getPersistenceType() {
	+ return $this->persistenceType;
	+ }
	+
	+ public function getPersistenceTypeDisplayName() {
	+ $map = array(
	+ self::PERSISTENCE_DATA => pht('Data'),
	+ self::PERSISTENCE_CACHE => pht('Cache'),
	+ self::PERSISTENCE_INDEX => pht('Index'),
	+ );
	+
	+ $type = $this->getPersistenceType();
	+
	+ return idx($map, $type, $type);
	+ }
	+
	protected function getSubschemata() {
	// NOTE: Keys and columns may have the same name, so make sure we return
	// everything.

	return array_merge(
	array_values($this->columns),
	array_values($this->keys));
	}

	public function setCollation($collation) {
	$this->collation = $collation;
	return $this;
	}

	public function getCollation() {
	return $this->collation;
	}

	public function setEngine($engine) {
	$this->engine = $engine;
	return $this;
	}

	public function getEngine() {
	return $this->engine;
	}

	protected function compareToSimilarSchema(
	PhabricatorConfigStorageSchema $expect) {

	$issues = array();
	if ($this->getCollation() != $expect->getCollation()) {
	$issues[] = self::ISSUE_COLLATION;
	}

	if ($this->getEngine() != $expect->getEngine()) {
	$issues[] = self::ISSUE_ENGINE;
	}

	return $issues;
	}

	public function newEmptyClone() {
	$clone = clone $this;
	$clone->columns = array();
	$clone->keys = array();
	return $clone;
	}

	}
	diff --git a/src/applications/differential/storage/DifferentialSchemaSpec.php b/src/applications/differential/storage/DifferentialSchemaSpec.php
	index e376c7f4f8..7aa76fa822 100644
	--- a/src/applications/differential/storage/DifferentialSchemaSpec.php
	+++ b/src/applications/differential/storage/DifferentialSchemaSpec.php
	@@ -1,64 +1,67 @@
	<?php

	final class DifferentialSchemaSpec extends PhabricatorConfigSchemaSpec {

	public function buildSchemata() {
	$this->buildEdgeSchemata(new DifferentialRevision());

	$this->buildRawSchema(
	id(new DifferentialRevision())->getApplicationName(),
	DifferentialChangeset::TABLE_CACHE,
	array(
	'id' => 'id',
	'cache' => 'bytes',
	'dateCreated' => 'epoch',
	),
	array(
	'PRIMARY' => array(
	'columns' => array('id'),
	'unique' => true,
	),
	'dateCreated' => array(
	'columns' => array('dateCreated'),
	),
	+ ),
	+ array(
	+ 'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_CACHE,
	));

	$this->buildRawSchema(
	id(new DifferentialRevision())->getApplicationName(),
	DifferentialRevision::TABLE_COMMIT,
	array(
	'revisionID' => 'id',
	'commitPHID' => 'phid',
	),
	array(
	'PRIMARY' => array(
	'columns' => array('revisionID', 'commitPHID'),
	'unique' => true,
	),
	'commitPHID' => array(
	'columns' => array('commitPHID'),
	'unique' => true,
	),
	));

	$this->buildRawSchema(
	id(new DifferentialRevision())->getApplicationName(),
	ArcanistDifferentialRevisionHash::TABLE_NAME,
	array(
	'revisionID' => 'id',
	'type' => 'bytes4',
	'hash' => 'bytes40',
	),
	array(
	'type' => array(
	'columns' => array('type', 'hash'),
	),
	'revisionID' => array(
	'columns' => array('revisionID'),
	),
	));


	}

	}
	diff --git a/src/docs/user/configuration/configuring_backups.diviner b/src/docs/user/configuration/configuring_backups.diviner
	index 9776448216..d32eebb0da 100644
	--- a/src/docs/user/configuration/configuring_backups.diviner
	+++ b/src/docs/user/configuration/configuring_backups.diviner
	@@ -1,153 +1,171 @@
	@title Configuring Backups and Performing Migrations
	@group config

	Advice for backing up Phabricator, or migrating from one machine to another.


	Overview
	========

	Phabricator does not currently have a comprehensive backup system, but creating
	backups is not particularly difficult and Phabricator does have a few basic
	tools which can help you set up a reasonable process. In particular, the things
	which needs to be backed up are:

	- the MySQL databases;
	- hosted repositories;
	- uploaded files; and
	- your Phabricator configuration files.

	This document discusses approaches for backing up this data.

	If you are migrating from one machine to another, you can generally follow the
	same steps you would if you were creating a backup and then restoring it, you
	will just backup the old machine and then restore the data onto the new
	machine.

	WARNING: You need to restart Phabricator after restoring data.

	Restarting Phabricator after performing a restore makes sure that caches are
	flushed properly. For complete instructions, see
	@{article:Restarting Phabricator}.


	Backup: MySQL Databases
	=======================

	Most of Phabricator's data is stored in MySQL, and it's the most important thing
	to back up. You can run `bin/storage dump` to get a dump of all the MySQL
	databases. This is a convenience script which just runs a normal `mysqldump`,
	but will only dump databases Phabricator owns.

	Since most of this data is compressible, it may be helpful to run it through
	gzip prior to storage. For example:

	phabricator/ $ ./bin/storage dump \| gzip > backup.sql.gz

	Then store the backup somewhere safe, like in a box buried under an old tree
	stump. No one will ever think to look for it there.

	Restore: MySQL
	==============

	To restore a MySQL dump, just pipe it to `mysql` on a clean host. (You may need
	to uncompress it first, if you compressed it prior to storage.)

	$ gunzip -c backup.sql.gz \| mysql


	Backup: Hosted Repositories
	===========================

	If you host repositories in Phabricator, you should back them up. You can use
	`bin/repository list-paths` to show the local paths on disk for each
	repository. To back them up, copy them elsewhere.

	You can also just clone them and keep the clones up to date, or use
	{nav Add Mirror} to have the mirror somewhere automatically.


	Restore: Hosted Repositories
	============================

	To restore hosted repositories, copy them back into the correct locations
	as shown by `bin/repository list-paths`.


	Backup: Uploaded Files
	======================

	Uploaded files may be stored in several different locations. The backup
	procedure depends on where files are stored:

	Default / MySQL: Under the default configuration, uploaded files are stored
	in MySQL, so the MySQL backup will include all files. In this case, you don't
	need to do any additional work.

	Amazon S3: If you use Amazon S3, redundancy and backups are built in to the
	service. This is probably sufficient for most installs. If you trust Amazon with
	your data //except not really//, you can backup your S3 bucket outside of
	Phabricator.

	Local Disk: If you use the local disk storage engine, you'll need to back up
	files manually. You can do this by creating a copy of the root directory where
	you told Phabricator to put files (the `storage.local-disk.path` configuration
	setting).

	For more information about configuring how files are stored, see
	@{article:Configuring File Storage}.


	Restore: Uploaded Files
	=======================

	To restore a backup of local disk storage, just copy the backup into place.


	Backup: Configuration Files
	===========================

	You should also backup your configuration files, and any scripts you use to
	deploy or administrate Phabricator (like a customized upgrade script). The best
	way to do this is to check them into a private repository somewhere and just use
	whatever backup process you already have in place for repositories. Just copying
	them somewhere will work fine too, of course.

	In particular, you should backup this configuration file which Phabricator
	creates:

	phabricator/conf/local/local.json

	This file contains all of the configuration settings that have been adjusted
	by using `bin/config set <key> <value>`.


	Restore: Configuration Files
	============================

	To restore configuration files, just copy them into the right locations. Copy
	your backup of `local.json` to `phabricator/conf/local/local.json`.

	Security
	========

	MySQL dumps have no builtin encryption and most data in Phabricator is stored in
	a raw, accessible form, so giving a user access to backups is a lot like giving
	them shell access to the machine Phabricator runs on. In particular, a user who
	has the backups can:

	- read data that policies do not permit them to see;
	- read email addresses and object secret keys; and
	- read other users' session and conduit tokens and impersonate them.

	Some of this information is durable, so disclosure of even a very old backup may
	present a risk. If you restrict access to the Phabricator host or database, you
	should also restrict access to the backups.


	+Skipping Indexes
	+================
	+
	+By default, `bin/storage dump` does not dump all of the data in the database:
	+it skips some caches which can be rebuilt automatically and do not need to be
	+backed up. Some of these caches are very large, so the size of the dump may
	+be significantly smaller than the size of the databases.
	+
	+If you have a large amount of data, you can specify `--no-indexes` when taking
	+a database dump to skip additional tables which contain search indexes. This
	+will reduce the size (and increase the speed) of the backup. This is an
	+advanced option which most installs will not benefit from.
	+
	+This index data can be rebuilt after a restore, but will not be rebuilt
	+automatically. If you choose to use this flag, you must manually rebuild
	+indexes after a restore (for details, see ((reindex))).
	+
	+
	Next Steps
	==========

	Continue by:

	- returning to the @{article:Configuration Guide}.
	diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php
	index f491133c67..fecc0517e3 100644
	--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php
	+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php
	@@ -1,264 +1,317 @@
	<?php

	final class PhabricatorStorageManagementDumpWorkflow
	extends PhabricatorStorageManagementWorkflow {

	protected function didConstruct() {
	$this
	->setName('dump')
	->setExamples('dump [__options__]')
	->setSynopsis(pht('Dump all data in storage to stdout.'))
	->setArguments(
	array(
	array(
	'name' => 'for-replica',
	'help' => pht(
	'Add __--master-data__ to the __mysqldump__ command, '.
	'generating a CHANGE MASTER statement in the output.'),
	),
	array(
	'name' => 'output',
	'param' => 'file',
	'help' => pht(
	'Write output directly to disk. This handles errors better '.
	'than using pipes. Use with __--compress__ to gzip the '.
	'output.'),
	),
	array(
	'name' => 'compress',
	'help' => pht(
	'With __--output__, write a compressed file to disk instead '.
	'of a plaintext file.'),
	),
	+ array(
	+ 'name' => 'no-indexes',
	+ 'help' => pht(
	+ 'Do not dump data in rebuildable index tables. This means '.
	+ 'backups are smaller and faster, but you will need to manually '.
	+ 'rebuild indexes after performing a restore.'),
	+ ),
	array(
	'name' => 'overwrite',
	'help' => pht(
	'With __--output__, overwrite the output file if it already '.
	'exists.'),
	),
	));
	}

	protected function isReadOnlyWorkflow() {
	return true;
	}

	public function didExecute(PhutilArgumentParser $args) {
	$api = $this->getSingleAPI();
	$patches = $this->getPatches();

	$console = PhutilConsole::getConsole();

	+ $with_indexes = !$args->getArg('no-indexes');
	+
	$applied = $api->getAppliedPatches();
	if ($applied === null) {
	$namespace = $api->getNamespace();
	$console->writeErr(
	pht(
	'Storage Not Initialized: There is no database storage '.
	'initialized in this storage namespace ("%s"). Use '.
	'%s to initialize storage.',
	$namespace,
	'./bin/storage upgrade'));
	return 1;
	}

	$ref = $api->getRef();
	$ref_key = $ref->getRefKey();

	- $schemata_map = id(new PhabricatorConfigSchemaQuery())
	+ $schemata_query = id(new PhabricatorConfigSchemaQuery())
	->setAPIs(array($api))
	- ->setRefs(array($ref))
	- ->loadActualSchemata();
	- $schemata = $schemata_map[$ref_key];
	+ ->setRefs(array($ref));
	+
	+ $actual_map = $schemata_query->loadActualSchemata();
	+ $expect_map = $schemata_query->loadExpectedSchemata();
	+
	+ $schemata = $actual_map[$ref_key];
	+ $expect = $expect_map[$ref_key];

	$targets = array();
	foreach ($schemata->getDatabases() as $database_name => $database) {
	+ $expect_database = $expect->getDatabase($database_name);
	foreach ($database->getTables() as $table_name => $table) {
	+
	+ // NOTE: It's possible for us to find tables in these database which
	+ // we don't expect to be there. For example, an older version of
	+ // Phabricator may have had a table that was later dropped. We assume
	+ // these are data tables and always dump them, erring on the side of
	+ // caution.
	+
	+ $persistence = PhabricatorConfigTableSchema::PERSISTENCE_DATA;
	+ if ($expect_database) {
	+ $expect_table = $expect_database->getTable($table_name);
	+ if ($expect_table) {
	+ $persistence = $expect_table->getPersistenceType();
	+ }
	+ }
	+
	+ switch ($persistence) {
	+ case PhabricatorConfigTableSchema::PERSISTENCE_CACHE:
	+ // When dumping tables, leave the data in cache tables in the
	+ // database. This will be automatically rebuild after the data
	+ // is restored and does not need to be persisted in backups.
	+ $with_data = false;
	+ break;
	+ case PhabricatorConfigTableSchema::PERSISTENCE_INDEX:
	+ // When dumping tables, leave index data behind of the caller
	+ // specified "--no-indexes". These tables can be rebuilt manually
	+ // from other tables, but do not rebuild automatically.
	+ $with_data = $with_indexes;
	+ break;
	+ case PhabricatorConfigTableSchema::PERSISTENCE_DATA:
	+ default:
	+ $with_data = true;
	+ break;
	+ }
	+
	$targets[] = array(
	'database' => $database_name,
	'table' => $table_name,
	+ 'data' => $with_data,
	);
	}
	}

	list($host, $port) = $this->getBareHostAndPort($api->getHost());

	$has_password = false;

	$password = $api->getPassword();
	if ($password) {
	if (strlen($password->openEnvelope())) {
	$has_password = true;
	}
	}

	$output_file = $args->getArg('output');
	$is_compress = $args->getArg('compress');
	$is_overwrite = $args->getArg('overwrite');

	if ($is_compress) {
	if ($output_file === null) {
	throw new PhutilArgumentUsageException(
	pht(
	'The "--compress" flag can only be used alongside "--output".'));
	}
	}

	if ($is_overwrite) {
	if ($output_file === null) {
	throw new PhutilArgumentUsageException(
	pht(
	'The "--overwrite" flag can only be used alongside "--output".'));
	}
	}

	if ($output_file !== null) {
	if (Filesystem::pathExists($output_file)) {
	if (!$is_overwrite) {
	throw new PhutilArgumentUsageException(
	pht(
	'Output file "%s" already exists. Use "--overwrite" '.
	'to overwrite.',
	$output_file));
	}
	}
	}

	$argv = array();
	$argv[] = '--hex-blob';
	$argv[] = '--single-transaction';
	$argv[] = '--default-character-set=utf8';

	if ($args->getArg('for-replica')) {
	$argv[] = '--master-data';
	}

	$argv[] = '-u';
	$argv[] = $api->getUser();
	$argv[] = '-h';
	$argv[] = $host;

	if ($port) {
	$argv[] = '--port';
	$argv[] = $port;
	}

	$commands = array();
	foreach ($targets as $target) {
	$target_argv = $argv;

	+ if (!$target['data']) {
	+ $target_argv[] = '--no-data';
	+ }
	+
	if ($has_password) {
	$commands[] = csprintf(
	'mysqldump -p%P %Ls -- %R %R',
	$password,
	$target_argv,
	$target['database'],
	$target['table']);
	} else {
	$command = csprintf(
	'mysqldump %Ls -- %R %R',
	$target_argv,
	$target['database'],
	$target['table']);
	}

	$commands[] = $command;
	}


	// Decrease the CPU priority of this process so it doesn't contend with
	// other more important things.
	if (function_exists('proc_nice')) {
	proc_nice(19);
	}

	// If we are writing to a file, stream the command output to disk. This
	// mode makes sure the whole command fails if there's an error (commonly,
	// a full disk). See T6996 for discussion.

	if ($output_file === null) {
	$file = null;
	} else if ($is_compress) {
	$file = gzopen($output_file, 'wb1');
	} else {
	$file = fopen($output_file, 'wb');
	}

	if (!$file) {
	throw new Exception(
	pht(
	'Failed to open file "%s" for writing.',
	$file));
	}

	try {
	foreach ($commands as $command) {
	$future = new ExecFuture('%C', $command);

	$iterator = id(new FutureIterator(array($future)))
	->setUpdateInterval(0.100);
	foreach ($iterator as $ready) {
	list($stdout, $stderr) = $future->read();
	$future->discardBuffers();

	if (strlen($stderr)) {
	fwrite(STDERR, $stderr);
	}

	if (strlen($stdout)) {
	if (!$file) {
	$ok = fwrite(STDOUT, $stdout);
	} else if ($is_compress) {
	$ok = gzwrite($file, $stdout);
	} else {
	$ok = fwrite($file, $stdout);
	}

	if ($ok !== strlen($stdout)) {
	throw new Exception(
	pht(
	'Failed to write %d byte(s) to file "%s".',
	new PhutilNumber(strlen($stdout)),
	$output_file));
	}
	}

	if ($ready !== null) {
	$ready->resolvex();
	}
	}
	}

	if (!$file) {
	$ok = true;
	} else if ($is_compress) {
	$ok = gzclose($file);
	} else {
	$ok = fclose($file);
	}

	if ($ok !== true) {
	throw new Exception(
	pht(
	'Failed to close file "%s".',
	$output_file));
	}
	} catch (Exception $ex) {
	// If we might have written a partial file to disk, try to remove it so
	// we don't leave any confusing artifacts laying around.

	try {
	if ($file !== null) {
	Filesystem::remove($output_file);
	}
	} catch (Exception $ex) {
	// Ignore any errors we hit.
	}

	throw $ex;
	}

	return 0;
	}

	}

File Metadata

Mime Type: text/x-diff
Expires: Thu, May 1, 1:35 PM (1 d, 18 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 109000
Default Alt Text: (71 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions