Page MenuHomestyx hydra
Files F154473

No OneTemporary
Actions

View Options

diff --git a/src/applications/diviner/publisher/DivinerPublisher.php b/src/applications/diviner/publisher/DivinerPublisher.php
index 298d0b42e7..0c2c0b1ad1 100644
--- a/src/applications/diviner/publisher/DivinerPublisher.php
+++ b/src/applications/diviner/publisher/DivinerPublisher.php
@@ -1,156 +1,157 @@
<?php
abstract class DivinerPublisher {
private $atomCache;
private $atomGraphHashToNodeHashMap;
private $atomMap = array();
private $renderer;
private $config;
private $symbolReverseMap;
private $dropCaches;
public function setDropCaches($drop_caches) {
$this->dropCaches = $drop_caches;
return $this;
}
public function setRenderer(DivinerRenderer $renderer) {
$renderer->setPublisher($this);
$this->renderer = $renderer;
return $this;
}
public function getRenderer() {
return $this->renderer;
}
public function setConfig(array $config) {
$this->config = $config;
return $this;
}
public function getConfig($key, $default = null) {
return idx($this->config, $key, $default);
}
public function getConfigurationData() {
return $this->config;
}
public function setAtomCache(DivinerAtomCache $cache) {
$this->atomCache = $cache;
$graph_map = $this->atomCache->getGraphMap();
$this->atomGraphHashToNodeHashMap = array_flip($graph_map);
+ return $this;
}
protected function getAtomFromGraphHash($graph_hash) {
if (empty($this->atomGraphHashToNodeHashMap[$graph_hash])) {
throw new Exception("No such atom '{$graph_hash}'!");
}
return $this->getAtomFromNodeHash(
$this->atomGraphHashToNodeHashMap[$graph_hash]);
}
protected function getAtomFromNodeHash($node_hash) {
if (empty($this->atomMap[$node_hash])) {
$dict = $this->atomCache->getAtom($node_hash);
$this->atomMap[$node_hash] = DivinerAtom::newFromDictionary($dict);
}
return $this->atomMap[$node_hash];
}
protected function getSimilarAtoms(DivinerAtom $atom) {
if ($this->symbolReverseMap === null) {
$rmap = array();
$smap = $this->atomCache->getSymbolMap();
foreach ($smap as $nhash => $shash) {
$rmap[$shash][$nhash] = true;
}
$this->symbolReverseMap = $rmap;
}
$shash = $atom->getRef()->toHash();
if (empty($this->symbolReverseMap[$shash])) {
throw new Exception('Atom has no symbol map entry!');
}
$hashes = $this->symbolReverseMap[$shash];
$atoms = array();
foreach ($hashes as $hash => $ignored) {
$atoms[] = $this->getAtomFromNodeHash($hash);
}
$atoms = msort($atoms, 'getSortKey');
return $atoms;
}
/**
* If a book contains multiple definitions of some atom, like some function
* "f()", we assign them an arbitrary (but fairly stable) order and publish
* them as "function/f/1/", "function/f/2/", etc., or similar.
*/
protected function getAtomSimilarIndex(DivinerAtom $atom) {
$atoms = $this->getSimilarAtoms($atom);
if (count($atoms) == 1) {
return 0;
}
$index = 1;
foreach ($atoms as $similar_atom) {
if ($atom === $similar_atom) {
return $index;
}
$index++;
}
throw new Exception('Expected to find atom while disambiguating!');
}
abstract protected function loadAllPublishedHashes();
abstract protected function deleteDocumentsByHash(array $hashes);
abstract protected function createDocumentsByHash(array $hashes);
abstract public function findAtomByRef(DivinerAtomRef $ref);
final public function publishAtoms(array $hashes) {
$existing = $this->loadAllPublishedHashes();
if ($this->dropCaches) {
$deleted = $existing;
$created = $hashes;
} else {
$existing_map = array_fill_keys($existing, true);
$hashes_map = array_fill_keys($hashes, true);
$deleted = array_diff_key($existing_map, $hashes_map);
$created = array_diff_key($hashes_map, $existing_map);
$deleted = array_keys($deleted);
$created = array_keys($created);
}
echo pht('Deleting %d documents.', count($deleted))."\n";
$this->deleteDocumentsByHash($deleted);
echo pht('Creating %d documents.', count($created))."\n";
$this->createDocumentsByHash($created);
}
protected function shouldGenerateDocumentForAtom(DivinerAtom $atom) {
switch ($atom->getType()) {
case DivinerAtom::TYPE_METHOD:
case DivinerAtom::TYPE_FILE:
return false;
case DivinerAtom::TYPE_ARTICLE:
default:
break;
}
return true;
}
}
diff --git a/src/applications/diviner/workflow/DivinerGenerateWorkflow.php b/src/applications/diviner/workflow/DivinerGenerateWorkflow.php
index 9e1370d73b..2b0e4ed3ae 100644
--- a/src/applications/diviner/workflow/DivinerGenerateWorkflow.php
+++ b/src/applications/diviner/workflow/DivinerGenerateWorkflow.php
@@ -1,516 +1,537 @@
<?php
final class DivinerGenerateWorkflow extends DivinerWorkflow {
private $atomCache;
protected function didConstruct() {
$this
->setName('generate')
->setSynopsis(pht('Generate documentation.'))
->setArguments(
array(
array(
'name' => 'clean',
'help' => pht('Clear the caches before generating documentation.'),
),
array(
'name' => 'book',
'param' => 'path',
'help' => pht('Path to a Diviner book configuration.'),
),
+ array(
+ 'name' => 'publisher',
+ 'param' => 'class',
+ 'help' => pht('Specify a subclass of %s.', 'DivinerPublisher'),
+ 'default' => 'DivinerLivePublisher',
+ ),
));
}
protected function getAtomCache() {
if (!$this->atomCache) {
$book_root = $this->getConfig('root');
$book_name = $this->getConfig('name');
$cache_directory = $book_root.'/.divinercache/'.$book_name;
$this->atomCache = new DivinerAtomCache($cache_directory);
}
return $this->atomCache;
}
protected function log($message) {
$console = PhutilConsole::getConsole();
$console->writeErr($message."\n");
}
public function execute(PhutilArgumentParser $args) {
$book = $args->getArg('book');
if ($book) {
$books = array($book);
} else {
$cwd = getcwd();
$this->log(pht('FINDING DOCUMENTATION BOOKS'));
$books = id(new FileFinder($cwd))
->withType('f')
->withSuffix('book')
->find();
if (!$books) {
throw new PhutilArgumentUsageException(
pht(
"There are no Diviner '%s' files anywhere beneath the current ".
"directory. Use '%s' to specify a documentation book to generate.",
'.book',
'--book <book>'));
} else {
$this->log(pht('Found %s book(s).', new PhutilNumber(count($books))));
}
}
foreach ($books as $book) {
$short_name = basename($book);
$this->log(pht('Generating book "%s"...', $short_name));
$this->generateBook($book, $args);
$this->log(pht('Completed generation of "%s".', $short_name)."\n");
}
}
private function generateBook($book, PhutilArgumentParser $args) {
$this->atomCache = null;
$this->readBookConfiguration($book);
if ($args->getArg('clean')) {
$this->log(pht('CLEARING CACHES'));
$this->getAtomCache()->delete();
$this->log(pht('Done.')."\n");
}
// The major challenge of documentation generation is one of dependency
// management. When regenerating documentation, we want to do the smallest
// amount of work we can, so that regenerating documentation after minor
// changes is quick.
//
// = ATOM CACHE =
//
// In the first stage, we find all the direct changes to source code since
// the last run. This stage relies on two data structures:
//
// - File Hash Map: `map<file_hash, node_hash>`
// - Atom Map: `map<node_hash, true>`
//
// First, we hash all the source files in the project to detect any which
// have changed since the previous run (i.e., their hash is not present in
// the File Hash Map). If a file's content hash appears in the map, it has
// not changed, so we don't need to reparse it.
//
// We break the contents of each file into "atoms", which represent a unit
// of source code (like a function, method, class or file). Each atom has a
// "node hash" based on the content of the atom: if a function definition
// changes, the node hash of the atom changes too. The primary output of
// the atom cache is a list of node hashes which exist in the project. This
// is the Atom Map. The node hash depends only on the definition of the atom
// and the atomizer implementation. It ends with an "N", for "node".
//
// (We need the Atom Map in addition to the File Hash Map because each file
// may have several atoms in it (e.g., multiple functions, or a class and
// its methods). The File Hash Map contains an exhaustive list of all atoms
// with type "file", but not child atoms of those top-level atoms.)
//
// = GRAPH CACHE =
//
// We now know which atoms exist, and can compare the Atom Map to some
// existing cache to figure out what has changed. However, this isn't
// sufficient to figure out which documentation actually needs to be
// regenerated, because atoms depend on other atoms. For example, if `B
// extends A` and the definition for `A` changes, we need to regenerate the
// documentation in `B`. Similarly, if `X` links to `Y` and `Y` changes, we
// should regenerate `X`. (In both these cases, the documentation for the
// connected atom may not actually change, but in some cases it will, and
// the extra work we need to do is generally very small compared to the
// size of the project.)
//
// To figure out which other nodes have changed, we compute a "graph hash"
// for each node. This hash combines the "node hash" with the node hashes
// of connected nodes. Our primary output is a list of graph hashes, which
// a documentation generator can use to easily determine what work needs
// to be done by comparing the list with a list of cached graph hashes,
// then generating documentation for new hashes and deleting documentation
// for missing hashes. The graph hash ends with a "G", for "graph".
//
// In this stage, we rely on three data structures:
//
// - Symbol Map: `map<node_hash, symbol_hash>`
// - Edge Map: `map<node_hash, list<symbol_hash>>`
// - Graph Map: `map<node_hash, graph_hash>`
//
// Calculating the graph hash requires several steps, because we need to
// figure out which nodes an atom is attached to. The atom contains symbolic
// references to other nodes by name (e.g., `extends SomeClass`) in the form
// of @{class:DivinerAtomRefs}. We can also build a symbolic reference for
// any atom from the atom itself. Each @{class:DivinerAtomRef} generates a
// symbol hash, which ends with an "S", for "symbol".
//
// First, we update the symbol map. We remove (and mark dirty) any symbols
// associated with node hashes which no longer exist (e.g., old/dead nodes).
// Second, we add (and mark dirty) any symbols associated with new nodes.
// We also add edges defined by new nodes to the graph.
//
// We initialize a list of dirty nodes to the list of new nodes, then find
// all nodes connected to dirty symbols and add them to the dirty node list.
// This list now contains every node with a new or changed graph hash.
//
// We walk the dirty list and compute the new graph hashes, adding them
// to the graph hash map. This Graph Map can then be passed to an actual
// documentation generator, which can compare the graph hashes to a list
// of already-generated graph hashes and easily assess which documents need
// to be regenerated and which can be deleted.
$this->buildAtomCache();
$this->buildGraphCache();
- $this->publishDocumentation($args->getArg('clean'));
+ $publisher_class = $args->getArg('publisher');
+ $symbols = id(new PhutilSymbolLoader())
+ ->setName($publisher_class)
+ ->setConcreteOnly(true)
+ ->setAncestorClass('DivinerPublisher')
+ ->selectAndLoadSymbols();
+ if (!$symbols) {
+ throw new Exception(
+ pht(
+ "Publisher class '%s' must be a concrete subclass of %s.",
+ $publisher_class,
+ 'DivinerPublisher'));
+ }
+ $publisher = newv($publisher_class, array());
+
+ $this->publishDocumentation($args->getArg('clean'), $publisher);
}
/* -( Atom Cache )--------------------------------------------------------- */
private function buildAtomCache() {
$this->log(pht('BUILDING ATOM CACHE'));
$file_hashes = $this->findFilesInProject();
$this->log(pht('Found %d file(s) in project.', count($file_hashes)));
$this->deleteDeadAtoms($file_hashes);
$atomize = $this->getFilesToAtomize($file_hashes);
$this->log(pht('Found %d unatomized, uncached file(s).', count($atomize)));
$file_atomizers = $this->getAtomizersForFiles($atomize);
$this->log(pht('Found %d file(s) to atomize.', count($file_atomizers)));
$futures = $this->buildAtomizerFutures($file_atomizers);
$this->log(pht('Atomizing %d file(s).', count($file_atomizers)));
if ($futures) {
$this->resolveAtomizerFutures($futures, $file_hashes);
$this->log(pht('Atomization complete.'));
} else {
$this->log(pht('Atom cache is up to date, no files to atomize.'));
}
$this->log(pht('Writing atom cache.'));
$this->getAtomCache()->saveAtoms();
$this->log(pht('Done.')."\n");
}
private function getAtomizersForFiles(array $files) {
$rules = $this->getRules();
$exclude = $this->getExclude();
$atomizers = array();
foreach ($files as $file) {
foreach ($exclude as $pattern) {
if (preg_match($pattern, $file)) {
continue 2;
}
}
foreach ($rules as $rule => $atomizer) {
$ok = preg_match($rule, $file);
if ($ok === false) {
throw new Exception(
pht("Rule '%s' is not a valid regular expression.", $rule));
}
if ($ok) {
$atomizers[$file] = $atomizer;
continue;
}
}
}
return $atomizers;
}
private function getRules() {
return $this->getConfig('rules', array(
'/\\.diviner$/' => 'DivinerArticleAtomizer',
'/\\.php$/' => 'DivinerPHPAtomizer',
));
}
private function getExclude() {
$exclude = (array)$this->getConfig('exclude', array());
return $exclude;
}
private function findFilesInProject() {
$raw_hashes = id(new FileFinder($this->getConfig('root')))
->excludePath('*/.*')
->withType('f')
->setGenerateChecksums(true)
->find();
$version = $this->getDivinerAtomWorldVersion();
$file_hashes = array();
foreach ($raw_hashes as $file => $md5_hash) {
$rel_file = Filesystem::readablePath($file, $this->getConfig('root'));
// We want the hash to change if the file moves or Diviner gets updated,
// not just if the file content changes. Derive a hash from everything
// we care about.
$file_hashes[$rel_file] = md5("{$rel_file}\0{$md5_hash}\0{$version}").'F';
}
return $file_hashes;
}
private function deleteDeadAtoms(array $file_hashes) {
$atom_cache = $this->getAtomCache();
$hash_to_file = array_flip($file_hashes);
foreach ($atom_cache->getFileHashMap() as $hash => $atom) {
if (empty($hash_to_file[$hash])) {
$atom_cache->deleteFileHash($hash);
}
}
}
private function getFilesToAtomize(array $file_hashes) {
$atom_cache = $this->getAtomCache();
$atomize = array();
foreach ($file_hashes as $file => $hash) {
if (!$atom_cache->fileHashExists($hash)) {
$atomize[] = $file;
}
}
return $atomize;
}
private function buildAtomizerFutures(array $file_atomizers) {
$atomizers = array();
foreach ($file_atomizers as $file => $atomizer) {
$atomizers[$atomizer][] = $file;
}
$root = dirname(phutil_get_library_root('phabricator'));
$config_root = $this->getConfig('root');
$bar = id(new PhutilConsoleProgressBar())
->setTotal(count($file_atomizers));
$futures = array();
foreach ($atomizers as $class => $files) {
foreach (array_chunk($files, 32) as $chunk) {
$future = new ExecFuture(
'%s atomize --ugly --book %s --atomizer %s -- %Ls',
$root.'/bin/diviner',
$this->getBookConfigPath(),
$class,
$chunk);
$future->setCWD($config_root);
$futures[] = $future;
$bar->update(count($chunk));
}
}
$bar->done();
return $futures;
}
private function resolveAtomizerFutures(array $futures, array $file_hashes) {
assert_instances_of($futures, 'Future');
$atom_cache = $this->getAtomCache();
$bar = id(new PhutilConsoleProgressBar())
->setTotal(count($futures));
$futures = id(new FutureIterator($futures))
->limit(4);
foreach ($futures as $key => $future) {
try {
$atoms = $future->resolveJSON();
foreach ($atoms as $atom) {
if ($atom['type'] == DivinerAtom::TYPE_FILE) {
$file_hash = $file_hashes[$atom['file']];
$atom_cache->addFileHash($file_hash, $atom['hash']);
}
$atom_cache->addAtom($atom);
}
} catch (Exception $e) {
phlog($e);
}
$bar->update(1);
}
$bar->done();
}
/**
* Get a global version number, which changes whenever any atom or atomizer
* implementation changes in a way which is not backward-compatible.
*/
private function getDivinerAtomWorldVersion() {
$version = array();
$version['atom'] = DivinerAtom::getAtomSerializationVersion();
$version['rules'] = $this->getRules();
$atomizers = id(new PhutilSymbolLoader())
->setAncestorClass('DivinerAtomizer')
->setConcreteOnly(true)
->selectAndLoadSymbols();
$atomizer_versions = array();
foreach ($atomizers as $atomizer) {
$atomizer_versions[$atomizer['name']] = call_user_func(
array(
$atomizer['name'],
'getAtomizerVersion',
));
}
ksort($atomizer_versions);
$version['atomizers'] = $atomizer_versions;
return md5(serialize($version));
}
/* -( Graph Cache )-------------------------------------------------------- */
private function buildGraphCache() {
$this->log(pht('BUILDING GRAPH CACHE'));
$atom_cache = $this->getAtomCache();
$symbol_map = $atom_cache->getSymbolMap();
$atoms = $atom_cache->getAtomMap();
$dirty_symbols = array();
$dirty_nhashes = array();
$del_atoms = array_diff_key($symbol_map, $atoms);
$this->log(pht('Found %d obsolete atom(s) in graph.', count($del_atoms)));
foreach ($del_atoms as $nhash => $shash) {
$atom_cache->deleteSymbol($nhash);
$dirty_symbols[$shash] = true;
$atom_cache->deleteEdges($nhash);
$atom_cache->deleteGraph($nhash);
}
$new_atoms = array_diff_key($atoms, $symbol_map);
$this->log(pht('Found %d new atom(s) in graph.', count($new_atoms)));
foreach ($new_atoms as $nhash => $ignored) {
$shash = $this->computeSymbolHash($nhash);
$atom_cache->addSymbol($nhash, $shash);
$dirty_symbols[$shash] = true;
$atom_cache->addEdges($nhash, $this->getEdges($nhash));
$dirty_nhashes[$nhash] = true;
}
$this->log(pht('Propagating changes through the graph.'));
// Find all the nodes which point at a dirty node, and dirty them. Then
// find all the nodes which point at those nodes and dirty them, and so
// on. (This is slightly overkill since we probably don't need to propagate
// dirtiness across documentation "links" between symbols, but we do want
// to propagate it across "extends", and we suffer only a little bit of
// collateral damage by over-dirtying as long as the documentation isn't
// too well-connected.)
$symbol_stack = array_keys($dirty_symbols);
while ($symbol_stack) {
$symbol_hash = array_pop($symbol_stack);
foreach ($atom_cache->getEdgesWithDestination($symbol_hash) as $edge) {
$dirty_nhashes[$edge] = true;
$src_hash = $this->computeSymbolHash($edge);
if (empty($dirty_symbols[$src_hash])) {
$dirty_symbols[$src_hash] = true;
$symbol_stack[] = $src_hash;
}
}
}
$this->log(pht('Found %d affected atoms.', count($dirty_nhashes)));
foreach ($dirty_nhashes as $nhash => $ignored) {
$atom_cache->addGraph($nhash, $this->computeGraphHash($nhash));
}
$this->log(pht('Writing graph cache.'));
$atom_cache->saveGraph();
$atom_cache->saveEdges();
$atom_cache->saveSymbols();
$this->log(pht('Done.')."\n");
}
private function computeSymbolHash($node_hash) {
$atom_cache = $this->getAtomCache();
$atom = $atom_cache->getAtom($node_hash);
if (!$atom) {
throw new Exception(
pht("No such atom with node hash '%s'!", $node_hash));
}
$ref = DivinerAtomRef::newFromDictionary($atom['ref']);
return $ref->toHash();
}
private function getEdges($node_hash) {
$atom_cache = $this->getAtomCache();
$atom = $atom_cache->getAtom($node_hash);
$refs = array();
// Make the atom depend on its own symbol, so that all atoms with the same
// symbol are dirtied (e.g., if a codebase defines the function `f()`
// several times, all of them should be dirtied when one is dirtied).
$refs[DivinerAtomRef::newFromDictionary($atom)->toHash()] = true;
foreach (array_merge($atom['extends'], $atom['links']) as $ref_dict) {
$ref = DivinerAtomRef::newFromDictionary($ref_dict);
if ($ref->getBook() == $atom['book']) {
$refs[$ref->toHash()] = true;
}
}
return array_keys($refs);
}
private function computeGraphHash($node_hash) {
$atom_cache = $this->getAtomCache();
$atom = $atom_cache->getAtom($node_hash);
$edges = $this->getEdges($node_hash);
sort($edges);
$inputs = array(
'atomHash' => $atom['hash'],
'edges' => $edges,
);
return md5(serialize($inputs)).'G';
}
- private function publishDocumentation($clean) {
+ private function publishDocumentation($clean, DivinerPublisher $publisher) {
$atom_cache = $this->getAtomCache();
$graph_map = $atom_cache->getGraphMap();
$this->log(pht('PUBLISHING DOCUMENTATION'));
- $publisher = new DivinerLivePublisher();
- $publisher->setDropCaches($clean);
- $publisher->setConfig($this->getAllConfig());
- $publisher->setAtomCache($atom_cache);
- $publisher->setRenderer(new DivinerDefaultRenderer());
- $publisher->publishAtoms(array_values($graph_map));
+ $publisher
+ ->setDropCaches($clean)
+ ->setConfig($this->getAllConfig())
+ ->setAtomCache($atom_cache)
+ ->setRenderer(new DivinerDefaultRenderer())
+ ->publishAtoms(array_values($graph_map));
$this->log(pht('Done.'));
}
}

File Metadata

Mime Type
text/x-diff
Expires
Wed, May 21, 11:57 AM (1 d, 15 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
126242
Default Alt Text
(23 KB)

Event Timeline

styx copyright 2020 sirocyl / ellie (kymkdd) / ncommander / styx contributors
Use of other names and brands is not intended to reflect a claim of, or right to, use the name or brand.
styx is an operating system using the Linux kernel and components, and "styx Linux" as a phrase does not reflect the inclusion of the Linux brand or trademark in the styx project.
Linux® is the registered trademark of Linus Torvalds in the U.S. and other countries.
Terms and Conditions (TODO: fix this!)