Page MenuHomestyx hydra
Files F207068

No OneTemporary
Actions

View Options

diff --git a/src/applications/files/management/PhabricatorFilesManagementCompactWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementCompactWorkflow.php
index 0c8e7153e5..5d8115389b 100644
--- a/src/applications/files/management/PhabricatorFilesManagementCompactWorkflow.php
+++ b/src/applications/files/management/PhabricatorFilesManagementCompactWorkflow.php
@@ -1,134 +1,118 @@
<?php
final class PhabricatorFilesManagementCompactWorkflow
extends PhabricatorFilesManagementWorkflow {
protected function didConstruct() {
+ $arguments = $this->newIteratorArguments();
+ $arguments[] = array(
+ 'name' => 'dry-run',
+ 'help' => pht('Show what would be compacted.'),
+ );
+
$this
->setName('compact')
->setSynopsis(
pht(
'Merge identical files to share the same storage. In some cases, '.
'this can repair files with missing data.'))
- ->setArguments(
- array(
- array(
- 'name' => 'dry-run',
- 'help' => pht('Show what would be compacted.'),
- ),
- array(
- 'name' => 'all',
- 'help' => pht('Compact all files.'),
- ),
- array(
- 'name' => 'names',
- 'wildcard' => true,
- ),
- ));
+ ->setArguments($arguments);
}
public function execute(PhutilArgumentParser $args) {
$console = PhutilConsole::getConsole();
$iterator = $this->buildIterator($args);
- if (!$iterator) {
- throw new PhutilArgumentUsageException(
- pht(
- 'Either specify a list of files to compact, or use `%s` '.
- 'to compact all files.',
- '--all'));
- }
-
$is_dry_run = $args->getArg('dry-run');
foreach ($iterator as $file) {
$monogram = $file->getMonogram();
$hash = $file->getContentHash();
if (!$hash) {
$console->writeOut(
"%s\n",
pht('%s: No content hash.', $monogram));
continue;
}
// Find other files with the same content hash. We're going to point
// them at the data for this file.
$similar_files = id(new PhabricatorFile())->loadAllWhere(
'contentHash = %s AND id != %d AND
(storageEngine != %s OR storageHandle != %s)',
$hash,
$file->getID(),
$file->getStorageEngine(),
$file->getStorageHandle());
if (!$similar_files) {
$console->writeOut(
"%s\n",
pht('%s: No other files with the same content hash.', $monogram));
continue;
}
// Only compact files into this one if we can load the data. This
// prevents us from breaking working files if we're missing some data.
try {
$data = $file->loadFileData();
} catch (Exception $ex) {
$data = null;
}
if ($data === null) {
$console->writeOut(
"%s\n",
pht(
'%s: Unable to load file data; declining to compact.',
$monogram));
continue;
}
foreach ($similar_files as $similar_file) {
if ($is_dry_run) {
$console->writeOut(
"%s\n",
pht(
'%s: Would compact storage with %s.',
$monogram,
$similar_file->getMonogram()));
continue;
}
$console->writeOut(
"%s\n",
pht(
'%s: Compacting storage with %s.',
$monogram,
$similar_file->getMonogram()));
$old_instance = null;
try {
$old_instance = $similar_file->instantiateStorageEngine();
$old_engine = $similar_file->getStorageEngine();
$old_handle = $similar_file->getStorageHandle();
} catch (Exception $ex) {
// If the old stuff is busted, we just won't try to delete the
// old data.
phlog($ex);
}
$similar_file
->setStorageEngine($file->getStorageEngine())
->setStorageHandle($file->getStorageHandle())
->save();
if ($old_instance) {
$similar_file->deleteFileDataIfUnused(
$old_instance,
$old_engine,
$old_handle);
}
}
}
return 0;
}
}
diff --git a/src/applications/files/management/PhabricatorFilesManagementCycleWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementCycleWorkflow.php
index 6d574d633a..cbcfac2cc8 100644
--- a/src/applications/files/management/PhabricatorFilesManagementCycleWorkflow.php
+++ b/src/applications/files/management/PhabricatorFilesManagementCycleWorkflow.php
@@ -1,132 +1,118 @@
<?php
final class PhabricatorFilesManagementCycleWorkflow
extends PhabricatorFilesManagementWorkflow {
protected function didConstruct() {
+ $arguments = $this->newIteratorArguments();
+ $arguments[] = array(
+ 'name' => 'key',
+ 'param' => 'keyname',
+ 'help' => pht('Select a specific storage key to cycle to.'),
+ );
+
$this
->setName('cycle')
->setSynopsis(
pht('Cycle master key for encrypted files.'))
- ->setArguments(
- array(
- array(
- 'name' => 'key',
- 'param' => 'keyname',
- 'help' => pht('Select a specific storage key to cycle to.'),
- ),
- array(
- 'name' => 'all',
- 'help' => pht('Change encoding for all files.'),
- ),
- array(
- 'name' => 'names',
- 'wildcard' => true,
- ),
- ));
+ ->setArguments($arguments);
}
public function execute(PhutilArgumentParser $args) {
$iterator = $this->buildIterator($args);
- if (!$iterator) {
- throw new PhutilArgumentUsageException(
- pht(
- 'Either specify a list of files to cycle, or use --all to cycle '.
- 'all files.'));
- }
$format_map = PhabricatorFileStorageFormat::getAllFormats();
$engines = PhabricatorFileStorageEngine::loadAllEngines();
$key_name = $args->getArg('key');
$failed = array();
foreach ($iterator as $file) {
$monogram = $file->getMonogram();
$engine_key = $file->getStorageEngine();
$engine = idx($engines, $engine_key);
if (!$engine) {
echo tsprintf(
"%s\n",
pht(
'%s: Uses unknown storage engine "%s".',
$monogram,
$engine_key));
$failed[] = $file;
continue;
}
if ($engine->isChunkEngine()) {
echo tsprintf(
"%s\n",
pht(
'%s: Stored as chunks, declining to cycle directly.',
$monogram));
continue;
}
$format_key = $file->getStorageFormat();
if (empty($format_map[$format_key])) {
echo tsprintf(
"%s\n",
pht(
'%s: Uses unknown storage format "%s".',
$monogram,
$format_key));
$failed[] = $file;
continue;
}
$format = clone $format_map[$format_key];
$format->setFile($file);
if (!$format->canCycleMasterKey()) {
echo tsprintf(
"%s\n",
pht(
'%s: Storage format ("%s") does not support key cycling.',
$monogram,
$format->getStorageFormatName()));
continue;
}
echo tsprintf(
"%s\n",
pht(
'%s: Cycling master key.',
$monogram));
try {
if ($key_name) {
$format->selectMasterKey($key_name);
}
$file->cycleMasterStorageKey($format);
echo tsprintf(
"%s\n",
pht('Done.'));
} catch (Exception $ex) {
echo tsprintf(
"%B\n",
pht('Failed! %s', (string)$ex));
$failed[] = $file;
}
}
if ($failed) {
$monograms = mpull($failed, 'getMonogram');
echo tsprintf(
"%s\n",
pht('Failures: %s.', implode(', ', $monograms)));
return 1;
}
return 0;
}
}
diff --git a/src/applications/files/management/PhabricatorFilesManagementEncodeWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementEncodeWorkflow.php
index 1d972326da..f7d299bc5f 100644
--- a/src/applications/files/management/PhabricatorFilesManagementEncodeWorkflow.php
+++ b/src/applications/files/management/PhabricatorFilesManagementEncodeWorkflow.php
@@ -1,151 +1,140 @@
<?php
final class PhabricatorFilesManagementEncodeWorkflow
extends PhabricatorFilesManagementWorkflow {
protected function didConstruct() {
+ $arguments = $this->newIteratorArguments();
+
+ $arguments[] = array(
+ 'name' => 'as',
+ 'param' => 'format',
+ 'help' => pht('Select the storage format to use.'),
+ );
+
+ $arguments[] = array(
+ 'name' => 'key',
+ 'param' => 'keyname',
+ 'help' => pht('Select a specific storage key.'),
+ );
+
+ $arguments[] = array(
+ 'name' => 'force',
+ 'help' => pht(
+ 'Re-encode files which are already stored in the target '.
+ 'encoding.'),
+ );
+
$this
->setName('encode')
->setSynopsis(
pht('Change the storage encoding of files.'))
- ->setArguments(
- array(
- array(
- 'name' => 'as',
- 'param' => 'format',
- 'help' => pht('Select the storage format to use.'),
- ),
- array(
- 'name' => 'key',
- 'param' => 'keyname',
- 'help' => pht('Select a specific storage key.'),
- ),
- array(
- 'name' => 'all',
- 'help' => pht('Change encoding for all files.'),
- ),
- array(
- 'name' => 'force',
- 'help' => pht(
- 'Re-encode files which are already stored in the target '.
- 'encoding.'),
- ),
- array(
- 'name' => 'names',
- 'wildcard' => true,
- ),
- ));
+ ->setArguments($arguments);
}
public function execute(PhutilArgumentParser $args) {
$iterator = $this->buildIterator($args);
- if (!$iterator) {
- throw new PhutilArgumentUsageException(
- pht(
- 'Either specify a list of files to encode, or use --all to '.
- 'encode all files.'));
- }
$force = (bool)$args->getArg('force');
$format_list = PhabricatorFileStorageFormat::getAllFormats();
$format_list = array_keys($format_list);
$format_list = implode(', ', $format_list);
$format_key = $args->getArg('as');
if (!strlen($format_key)) {
throw new PhutilArgumentUsageException(
pht(
'Use --as <format> to select a target encoding format. Available '.
'formats are: %s.',
$format_list));
}
$format = PhabricatorFileStorageFormat::getFormat($format_key);
if (!$format) {
throw new PhutilArgumentUsageException(
pht(
'Storage format "%s" is not valid. Available formats are: %s.',
$format_key,
$format_list));
}
$key_name = $args->getArg('key');
if (strlen($key_name)) {
$format->selectMasterKey($key_name);
}
$engines = PhabricatorFileStorageEngine::loadAllEngines();
$failed = array();
foreach ($iterator as $file) {
$monogram = $file->getMonogram();
$engine_key = $file->getStorageEngine();
$engine = idx($engines, $engine_key);
if (!$engine) {
echo tsprintf(
"%s\n",
pht(
'%s: Uses unknown storage engine "%s".',
$monogram,
$engine_key));
$failed[] = $file;
continue;
}
if ($engine->isChunkEngine()) {
echo tsprintf(
"%s\n",
pht(
'%s: Stored as chunks, no data to encode directly.',
$monogram));
continue;
}
if (($file->getStorageFormat() == $format_key) && !$force) {
echo tsprintf(
"%s\n",
pht(
'%s: Already encoded in target format.',
$monogram));
continue;
}
echo tsprintf(
"%s\n",
pht(
'%s: Changing encoding from "%s" to "%s".',
$monogram,
$file->getStorageFormat(),
$format_key));
try {
$file->migrateToStorageFormat($format);
echo tsprintf(
"%s\n",
pht('Done.'));
} catch (Exception $ex) {
echo tsprintf(
"%B\n",
pht('Failed! %s', (string)$ex));
$failed[] = $file;
}
}
if ($failed) {
$monograms = mpull($failed, 'getMonogram');
echo tsprintf(
"%s\n",
pht('Failures: %s.', implode(', ', $monograms)));
return 1;
}
return 0;
}
}
diff --git a/src/applications/files/management/PhabricatorFilesManagementIntegrityWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementIntegrityWorkflow.php
index 344df71460..a30f4f970b 100644
--- a/src/applications/files/management/PhabricatorFilesManagementIntegrityWorkflow.php
+++ b/src/applications/files/management/PhabricatorFilesManagementIntegrityWorkflow.php
@@ -1,325 +1,317 @@
<?php
final class PhabricatorFilesManagementIntegrityWorkflow
extends PhabricatorFilesManagementWorkflow {
protected function didConstruct() {
+ $arguments = $this->newIteratorArguments();
+
+ $arguments[] = array(
+ 'name' => 'strip',
+ 'help' => pht(
+ 'DANGEROUS. Strip integrity hashes from files. This makes '.
+ 'files vulnerable to corruption or tampering.'),
+ );
+
+ $arguments[] = array(
+ 'name' => 'corrupt',
+ 'help' => pht(
+ 'Corrupt integrity hashes for given files. This is intended '.
+ 'for debugging.'),
+ );
+
+ $arguments[] = array(
+ 'name' => 'compute',
+ 'help' => pht(
+ 'Compute and update integrity hashes for files which do not '.
+ 'yet have them.'),
+ );
+
+ $arguments[] = array(
+ 'name' => 'overwrite',
+ 'help' => pht(
+ 'DANGEROUS. Recompute and update integrity hashes, overwriting '.
+ 'invalid hashes. This may mark corrupt or dangerous files as '.
+ 'valid.'),
+ );
+
+ $arguments[] = array(
+ 'name' => 'force',
+ 'short' => 'f',
+ 'help' => pht(
+ 'Execute dangerous operations without prompting for '.
+ 'confirmation.'),
+ );
+
+
$this
->setName('integrity')
->setSynopsis(pht('Verify or recalculate file integrity hashes.'))
- ->setArguments(
- array(
- array(
- 'name' => 'all',
- 'help' => pht('Affect all files.'),
- ),
- array(
- 'name' => 'strip',
- 'help' => pht(
- 'DANGEROUS. Strip integrity hashes from files. This makes '.
- 'files vulnerable to corruption or tampering.'),
- ),
- array(
- 'name' => 'corrupt',
- 'help' => pht(
- 'Corrupt integrity hashes for given files. This is intended '.
- 'for debugging.'),
- ),
- array(
- 'name' => 'compute',
- 'help' => pht(
- 'Compute and update integrity hashes for files which do not '.
- 'yet have them.'),
- ),
- array(
- 'name' => 'overwrite',
- 'help' => pht(
- 'DANGEROUS. Recompute and update integrity hashes, overwriting '.
- 'invalid hashes. This may mark corrupt or dangerous files as '.
- 'valid.'),
- ),
- array(
- 'name' => 'force',
- 'short' => 'f',
- 'help' => pht(
- 'Execute dangerous operations without prompting for '.
- 'confirmation.'),
- ),
- array(
- 'name' => 'names',
- 'wildcard' => true,
- ),
- ));
+ ->setArguments($arguments);
}
public function execute(PhutilArgumentParser $args) {
$modes = array();
$is_strip = $args->getArg('strip');
if ($is_strip) {
$modes[] = 'strip';
}
$is_corrupt = $args->getArg('corrupt');
if ($is_corrupt) {
$modes[] = 'corrupt';
}
$is_compute = $args->getArg('compute');
if ($is_compute) {
$modes[] = 'compute';
}
$is_overwrite = $args->getArg('overwrite');
if ($is_overwrite) {
$modes[] = 'overwrite';
}
$is_verify = !$modes;
if ($is_verify) {
$modes[] = 'verify';
}
if (count($modes) > 1) {
throw new PhutilArgumentUsageException(
pht(
'You have selected multiple operation modes (%s). Choose a '.
'single mode to operate in.',
implode(', ', $modes)));
}
$is_force = $args->getArg('force');
if (!$is_force) {
$prompt = null;
if ($is_strip) {
$prompt = pht(
'Stripping integrity hashes is dangerous and makes files '.
'vulnerable to corruption or tampering.');
}
if ($is_corrupt) {
$prompt = pht(
'Corrupting integrity hashes will prevent files from being '.
'accessed. This mode is intended only for development and '.
'debugging.');
}
if ($is_overwrite) {
$prompt = pht(
'Overwriting integrity hashes is dangerous and may mark files '.
'which have been corrupted or tampered with as safe.');
}
if ($prompt) {
$this->logWarn(pht('DANGEROUS'), $prompt);
if (!phutil_console_confirm(pht('Continue anyway?'))) {
throw new PhutilArgumentUsageException(pht('Aborted workflow.'));
}
}
}
$iterator = $this->buildIterator($args);
- if (!$iterator) {
- throw new PhutilArgumentUsageException(
- pht(
- 'Either specify a list of files to affect, or use "--all" to '.
- 'affect all files.'));
- }
$failure_count = 0;
$total_count = 0;
foreach ($iterator as $file) {
$total_count++;
$display_name = $file->getMonogram();
$old_hash = $file->getIntegrityHash();
if ($is_strip) {
if ($old_hash === null) {
$this->logInfo(
pht('SKIPPED'),
pht(
'File "%s" does not have an integrity hash to strip.',
$display_name));
} else {
$file
->setIntegrityHash(null)
->save();
$this->logWarn(
pht('STRIPPED'),
pht(
'Stripped integrity hash for "%s".',
$display_name));
}
continue;
}
$need_hash = ($is_verify && $old_hash) ||
($is_compute && ($old_hash === null)) ||
($is_corrupt) ||
($is_overwrite);
if ($need_hash) {
try {
$new_hash = $file->newIntegrityHash();
} catch (Exception $ex) {
$failure_count++;
$this->logFail(
pht('ERROR'),
pht(
'Unable to compute integrity hash for file "%s": %s',
$display_name,
$ex->getMessage()));
continue;
}
} else {
$new_hash = null;
}
// NOTE: When running in "corrupt" mode, we only corrupt the hash if
// we're able to compute a valid hash. Some files, like chunked files,
// do not support integrity hashing so corrupting them would create an
// unusual state.
if ($is_corrupt) {
if ($new_hash === null) {
$this->logInfo(
pht('IGNORED'),
pht(
'Storage for file "%s" does not support integrity hashing.',
$display_name));
} else {
$file
->setIntegrityHash('<corrupted>')
->save();
$this->logWarn(
pht('CORRUPTED'),
pht(
'Corrupted integrity hash for file "%s".',
$display_name));
}
continue;
}
if ($is_verify) {
if ($old_hash === null) {
$this->logInfo(
pht('NONE'),
pht(
'File "%s" has no stored integrity hash.',
$display_name));
} else if ($new_hash === null) {
$failure_count++;
$this->logWarn(
pht('UNEXPECTED'),
pht(
'Storage for file "%s" does not support integrity hashing, '.
'but the file has an integrity hash.',
$display_name));
} else if (phutil_hashes_are_identical($old_hash, $new_hash)) {
$this->logOkay(
pht('VALID'),
pht(
'File "%s" has a valid integrity hash.',
$display_name));
} else {
$failure_count++;
$this->logFail(
pht('MISMATCH'),
pht(
'File "%s" has an invalid integrity hash!',
$display_name));
}
continue;
}
if ($is_compute) {
if ($old_hash !== null) {
$this->logInfo(
pht('SKIP'),
pht(
'File "%s" already has an integrity hash.',
$display_name));
} else if ($new_hash === null) {
$this->logInfo(
pht('IGNORED'),
pht(
'Storage for file "%s" does not support integrity hashing.',
$display_name));
} else {
$file
->setIntegrityHash($new_hash)
->save();
$this->logOkay(
pht('COMPUTE'),
pht(
'Computed and stored integrity hash for file "%s".',
$display_name));
}
continue;
}
if ($is_overwrite) {
$same_hash = ($old_hash !== null) &&
($new_hash !== null) &&
phutil_hashes_are_identical($old_hash, $new_hash);
if ($new_hash === null) {
$this->logInfo(
pht('IGNORED'),
pht(
'Storage for file "%s" does not support integrity hashing.',
$display_name));
} else if ($same_hash) {
$this->logInfo(
pht('UNCHANGED'),
pht(
'File "%s" already has the correct integrity hash.',
$display_name));
} else {
$file
->setIntegrityHash($new_hash)
->save();
$this->logOkay(
pht('OVERWRITE'),
pht(
'Overwrote integrity hash for file "%s".',
$display_name));
}
continue;
}
}
if ($failure_count) {
$this->logFail(
pht('FAIL'),
pht(
'Processed %s file(s), encountered %s error(s).',
new PhutilNumber($total_count),
new PhutilNumber($failure_count)));
} else {
$this->logOkay(
pht('DONE'),
pht(
'Processed %s file(s) with no errors.',
new PhutilNumber($total_count)));
}
return 0;
}
}
diff --git a/src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php
index 58d5155aed..859ee0bf2d 100644
--- a/src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php
+++ b/src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php
@@ -1,281 +1,266 @@
<?php
final class PhabricatorFilesManagementMigrateWorkflow
extends PhabricatorFilesManagementWorkflow {
protected function didConstruct() {
+ $arguments = $this->newIteratorArguments();
+
+ $arguments[] = array(
+ 'name' => 'engine',
+ 'param' => 'storage-engine',
+ 'help' => pht('Migrate to the named storage engine.'),
+ );
+
+ $arguments[] = array(
+ 'name' => 'dry-run',
+ 'help' => pht('Show what would be migrated.'),
+ );
+
+ $arguments[] = array(
+ 'name' => 'min-size',
+ 'param' => 'bytes',
+ 'help' => pht(
+ 'Do not migrate data for files which are smaller than a given '.
+ 'filesize.'),
+ );
+
+ $arguments[] = array(
+ 'name' => 'max-size',
+ 'param' => 'bytes',
+ 'help' => pht(
+ 'Do not migrate data for files which are larger than a given '.
+ 'filesize.'),
+ );
+
+ $arguments[] = array(
+ 'name' => 'copy',
+ 'help' => pht(
+ 'Copy file data instead of moving it: after migrating, do not '.
+ 'remove the old data even if it is no longer referenced.'),
+ );
+
+ $arguments[] = array(
+ 'name' => 'local-disk-source',
+ 'param' => 'path',
+ 'help' => pht(
+ 'When migrating from a local disk source, use the specified '.
+ 'path as the root directory.'),
+ );
+
$this
->setName('migrate')
->setSynopsis(pht('Migrate files between storage engines.'))
- ->setArguments(
- array(
- array(
- 'name' => 'engine',
- 'param' => 'storage_engine',
- 'help' => pht('Migrate to the named storage engine.'),
- ),
- array(
- 'name' => 'dry-run',
- 'help' => pht('Show what would be migrated.'),
- ),
- array(
- 'name' => 'min-size',
- 'param' => 'bytes',
- 'help' => pht(
- 'Do not migrate data for files which are smaller than a given '.
- 'filesize.'),
- ),
- array(
- 'name' => 'max-size',
- 'param' => 'bytes',
- 'help' => pht(
- 'Do not migrate data for files which are larger than a given '.
- 'filesize.'),
- ),
- array(
- 'name' => 'all',
- 'help' => pht('Migrate all files.'),
- ),
- array(
- 'name' => 'copy',
- 'help' => pht(
- 'Copy file data instead of moving it: after migrating, do not '.
- 'remove the old data even if it is no longer referenced.'),
- ),
- array(
- 'name' => 'names',
- 'wildcard' => true,
- ),
- array(
- 'name' => 'from-engine',
- 'param' => 'engine',
- 'help' => pht('Migrate files from the named storage engine.'),
- ),
- array(
- 'name' => 'local-disk-source',
- 'param' => 'path',
- 'help' => pht(
- 'When migrating from a local disk source, use the specified '.
- 'path as the root directory.'),
- ),
- ));
+ ->setArguments($arguments);
}
public function execute(PhutilArgumentParser $args) {
// See T13306. This flag allows you to import files from a backup of
// local disk storage into some other engine. When the caller provides
// the flag, we override the local disk engine configuration and treat
// it as though it is configured to use the specified location.
$local_disk_source = $args->getArg('local-disk-source');
if (strlen($local_disk_source)) {
$path = Filesystem::resolvePath($local_disk_source);
try {
Filesystem::assertIsDirectory($path);
} catch (FilesystemException $ex) {
throw new PhutilArgumentUsageException(
pht(
'The "--local-disk-source" argument must point to a valid, '.
'readable directory on local disk.'));
}
$env = PhabricatorEnv::beginScopedEnv();
$env->overrideEnvConfig('storage.local-disk.path', $path);
}
$target_key = $args->getArg('engine');
if (!$target_key) {
throw new PhutilArgumentUsageException(
pht(
'Specify an engine to migrate to with `%s`. '.
'Use `%s` to get a list of engines.',
'--engine',
'files engines'));
}
$target_engine = PhabricatorFile::buildEngine($target_key);
$iterator = $this->buildIterator($args);
- if (!$iterator) {
- throw new PhutilArgumentUsageException(
- pht(
- 'Either specify a list of files to migrate, or use `%s` '.
- 'to migrate all files.',
- '--all'));
- }
-
$is_dry_run = $args->getArg('dry-run');
$min_size = (int)$args->getArg('min-size');
$max_size = (int)$args->getArg('max-size');
$is_copy = $args->getArg('copy');
$failed = array();
$engines = PhabricatorFileStorageEngine::loadAllEngines();
$total_bytes = 0;
$total_files = 0;
foreach ($iterator as $file) {
$monogram = $file->getMonogram();
// See T7148. When we export data for an instance, we copy all the data
// for Files from S3 into the database dump so that the database dump is
// a complete, standalone archive of all the data. In the general case,
// installs may have a similar process using "--copy" to create a more
// complete backup.
// When doing this, we may run into temporary files which have been
// deleted between the time we took the original dump and the current
// timestamp. These files can't be copied since the data no longer
// exists: the daemons on the live install already deleted it.
// Simply avoid this whole mess by declining to migrate expired temporary
// files. They're as good as dead anyway.
$ttl = $file->getTTL();
if ($ttl) {
if ($ttl < PhabricatorTime::getNow()) {
echo tsprintf(
"%s\n",
pht(
'%s: Skipping expired temporary file.',
$monogram));
continue;
}
}
$engine_key = $file->getStorageEngine();
$engine = idx($engines, $engine_key);
if (!$engine) {
echo tsprintf(
"%s\n",
pht(
'%s: Uses unknown storage engine "%s".',
$monogram,
$engine_key));
$failed[] = $file;
continue;
}
if ($engine->isChunkEngine()) {
echo tsprintf(
"%s\n",
pht(
'%s: Stored as chunks, no data to migrate directly.',
$monogram));
continue;
}
if ($engine_key === $target_key) {
echo tsprintf(
"%s\n",
pht(
'%s: Already stored in engine "%s".',
$monogram,
$target_key));
continue;
}
$byte_size = $file->getByteSize();
if ($min_size && ($byte_size < $min_size)) {
echo tsprintf(
"%s\n",
pht(
'%s: File size (%s) is smaller than minimum size (%s).',
$monogram,
phutil_format_bytes($byte_size),
phutil_format_bytes($min_size)));
continue;
}
if ($max_size && ($byte_size > $max_size)) {
echo tsprintf(
"%s\n",
pht(
'%s: File size (%s) is larger than maximum size (%s).',
$monogram,
phutil_format_bytes($byte_size),
phutil_format_bytes($max_size)));
continue;
}
if ($is_dry_run) {
echo tsprintf(
"%s\n",
pht(
'%s: (%s) Would migrate from "%s" to "%s" (dry run)...',
$monogram,
phutil_format_bytes($byte_size),
$engine_key,
$target_key));
} else {
echo tsprintf(
"%s\n",
pht(
'%s: (%s) Migrating from "%s" to "%s"...',
$monogram,
phutil_format_bytes($byte_size),
$engine_key,
$target_key));
}
try {
if ($is_dry_run) {
// Do nothing, this is a dry run.
} else {
$file->migrateToEngine($target_engine, $is_copy);
}
$total_files += 1;
$total_bytes += $byte_size;
echo tsprintf(
"%s\n",
pht('Done.'));
} catch (Exception $ex) {
echo tsprintf(
"%s\n",
pht('Failed! %s', (string)$ex));
$failed[] = $file;
throw $ex;
}
}
echo tsprintf(
"%s\n",
pht(
'Total Migrated Files: %s',
new PhutilNumber($total_files)));
echo tsprintf(
"%s\n",
pht(
'Total Migrated Bytes: %s',
phutil_format_bytes($total_bytes)));
if ($is_dry_run) {
echo tsprintf(
"%s\n",
pht(
'This was a dry run, so no real migrations were performed.'));
}
if ($failed) {
$monograms = mpull($failed, 'getMonogram');
echo tsprintf(
"%s\n",
pht('Failures: %s.', implode(', ', $monograms)));
return 1;
}
return 0;
}
}
diff --git a/src/applications/files/management/PhabricatorFilesManagementRebuildWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementRebuildWorkflow.php
index f7fc890ae4..e577a04d55 100644
--- a/src/applications/files/management/PhabricatorFilesManagementRebuildWorkflow.php
+++ b/src/applications/files/management/PhabricatorFilesManagementRebuildWorkflow.php
@@ -1,156 +1,144 @@
<?php
final class PhabricatorFilesManagementRebuildWorkflow
extends PhabricatorFilesManagementWorkflow {
protected function didConstruct() {
+ $arguments = $this->newIteratorArguments();
+
+ $arguments[] = array(
+ 'name' => 'dry-run',
+ 'help' => pht('Show what would be updated.'),
+ );
+
+ $arguments[] = array(
+ 'name' => 'rebuild-mime',
+ 'help' => pht('Rebuild MIME information.'),
+ );
+
+ $arguments[] = array(
+ 'name' => 'rebuild-dimensions',
+ 'help' => pht('Rebuild image dimension information.'),
+ );
+
$this
->setName('rebuild')
->setSynopsis(pht('Rebuild metadata of old files.'))
- ->setArguments(
- array(
- array(
- 'name' => 'all',
- 'help' => pht('Update all files.'),
- ),
- array(
- 'name' => 'dry-run',
- 'help' => pht('Show what would be updated.'),
- ),
- array(
- 'name' => 'rebuild-mime',
- 'help' => pht('Rebuild MIME information.'),
- ),
- array(
- 'name' => 'rebuild-dimensions',
- 'help' => pht('Rebuild image dimension information.'),
- ),
- array(
- 'name' => 'names',
- 'wildcard' => true,
- ),
- ));
+ ->setArguments($arguments);
}
public function execute(PhutilArgumentParser $args) {
$console = PhutilConsole::getConsole();
$iterator = $this->buildIterator($args);
- if (!$iterator) {
- throw new PhutilArgumentUsageException(
- pht(
- 'Either specify a list of files to update, or use `%s` '.
- 'to update all files.',
- '--all'));
- }
$update = array(
'mime' => $args->getArg('rebuild-mime'),
'dimensions' => $args->getArg('rebuild-dimensions'),
);
// If the user didn't select anything, rebuild everything.
if (!array_filter($update)) {
foreach ($update as $key => $ignored) {
$update[$key] = true;
}
}
$is_dry_run = $args->getArg('dry-run');
$failed = array();
foreach ($iterator as $file) {
$fid = 'F'.$file->getID();
if ($update['mime']) {
$tmp = new TempFile();
Filesystem::writeFile($tmp, $file->loadFileData());
$new_type = Filesystem::getMimeType($tmp);
if ($new_type == $file->getMimeType()) {
$console->writeOut(
"%s\n",
pht(
'%s: Mime type not changed (%s).',
$fid,
$new_type));
} else {
if ($is_dry_run) {
$console->writeOut(
"%s\n",
pht(
"%s: Would update Mime type: '%s' -> '%s'.",
$fid,
$file->getMimeType(),
$new_type));
} else {
$console->writeOut(
"%s\n",
pht(
"%s: Updating Mime type: '%s' -> '%s'.",
$fid,
$file->getMimeType(),
$new_type));
$file->setMimeType($new_type);
$file->save();
}
}
}
if ($update['dimensions']) {
if (!$file->isViewableImage()) {
$console->writeOut(
"%s\n",
pht('%s: Not an image file.', $fid));
continue;
}
$metadata = $file->getMetadata();
$image_width = idx($metadata, PhabricatorFile::METADATA_IMAGE_WIDTH);
$image_height = idx($metadata, PhabricatorFile::METADATA_IMAGE_HEIGHT);
if ($image_width && $image_height) {
$console->writeOut(
"%s\n",
pht('%s: Image dimensions already exist.', $fid));
continue;
}
if ($is_dry_run) {
$console->writeOut(
"%s\n",
pht('%s: Would update file dimensions (dry run)', $fid));
continue;
}
$console->writeOut(
pht('%s: Updating metadata... ', $fid));
try {
$file->updateDimensions();
$console->writeOut("%s\n", pht('Done.'));
} catch (Exception $ex) {
$console->writeOut("%s\n", pht('Failed!'));
$console->writeErr("%s\n", (string)$ex);
$failed[] = $file;
}
}
}
if ($failed) {
$console->writeOut("**%s**\n", pht('Failures!'));
$ids = array();
foreach ($failed as $file) {
$ids[] = 'F'.$file->getID();
}
$console->writeOut("%s\n", implode(', ', $ids));
return 1;
} else {
$console->writeOut("**%s**\n", pht('Success!'));
return 0;
}
return 0;
}
}
diff --git a/src/applications/files/management/PhabricatorFilesManagementWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementWorkflow.php
index 44d43dc66a..6debf9c711 100644
--- a/src/applications/files/management/PhabricatorFilesManagementWorkflow.php
+++ b/src/applications/files/management/PhabricatorFilesManagementWorkflow.php
@@ -1,72 +1,92 @@
<?php
abstract class PhabricatorFilesManagementWorkflow
extends PhabricatorManagementWorkflow {
+ protected function newIteratorArguments() {
+ return array(
+ array(
+ 'name' => 'all',
+ 'help' => pht('Operate on all files.'),
+ ),
+ array(
+ 'name' => 'names',
+ 'wildcard' => true,
+ ),
+ array(
+ 'name' => 'from-engine',
+ 'param' => 'storage-engine',
+ 'help' => pht('Operate on files stored in a specified engine.'),
+ ),
+ );
+ }
+
protected function buildIterator(PhutilArgumentParser $args) {
$viewer = $this->getViewer();
- $names = $args->getArg('names');
$is_all = $args->getArg('all');
+
+ $names = $args->getArg('names');
$from_engine = $args->getArg('from-engine');
$any_constraint = ($from_engine || $names);
if (!$is_all && !$any_constraint) {
throw new PhutilArgumentUsageException(
pht(
- 'Use "--all" to migrate all files, or choose files to migrate '.
- 'with "--names" or "--from-engine".'));
+ 'Specify which files to operate on, or use "--all" to operate on '.
+ 'all files.'));
}
if ($is_all && $any_constraint) {
throw new PhutilArgumentUsageException(
pht(
- 'You can not migrate all files with "--all" and also migrate only '.
- 'a subset of files with "--from-engine" or "--names".'));
+ 'You can not operate on all files with "--all" and also operate '.
+ 'on a subset of files by naming them explicitly or using '.
+ 'constraint flags like "--from-engine".'));
}
// If we're migrating specific named files, convert the names into IDs
// first.
$ids = null;
if ($names) {
$files = $this->loadFilesWithNames($names);
$ids = mpull($files, 'getID');
}
$query = id(new PhabricatorFileQuery())
->setViewer($viewer);
if ($ids) {
$query->withIDs($ids);
}
if ($from_engine) {
$query->withStorageEngines(array($from_engine));
}
return new PhabricatorQueryIterator($query);
}
protected function loadFilesWithNames(array $names) {
$query = id(new PhabricatorObjectQuery())
->setViewer($this->getViewer())
->withNames($names)
->withTypes(array(PhabricatorFileFilePHIDType::TYPECONST));
$query->execute();
$files = $query->getNamedResults();
foreach ($names as $name) {
if (empty($files[$name])) {
throw new PhutilArgumentUsageException(
pht(
'No file "%s" exists.',
$name));
}
}
return array_values($files);
}
}

File Metadata

Mime Type
text/x-diff
Expires
Wed, Jul 2, 3:11 AM (1 d, 14 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
164641
Default Alt Text
(41 KB)

Event Timeline

styx copyright 2020 sirocyl / ellie (kymkdd) / ncommander / styx contributors
Use of other names and brands is not intended to reflect a claim of, or right to, use the name or brand.
styx is an operating system using the Linux kernel and components, and "styx Linux" as a phrase does not reflect the inclusion of the Linux brand or trademark in the styx project.
Linux® is the registered trademark of Linus Torvalds in the U.S. and other countries.
Terms and Conditions (TODO: fix this!)