Page MenuHomestyx hydra
Files F83129

No OneTemporary
Actions

View Options

diff --git a/src/applications/conpherence/query/ConpherenceFulltextQuery.php b/src/applications/conpherence/query/ConpherenceFulltextQuery.php
index 43de7455ae..ba734049f8 100644
--- a/src/applications/conpherence/query/ConpherenceFulltextQuery.php
+++ b/src/applications/conpherence/query/ConpherenceFulltextQuery.php
@@ -1,85 +1,85 @@
<?php
final class ConpherenceFulltextQuery
extends PhabricatorOffsetPagedQuery {
private $threadPHIDs;
private $previousTransactionPHIDs;
private $fulltext;
public function withThreadPHIDs(array $phids) {
$this->threadPHIDs = $phids;
return $this;
}
public function withPreviousTransactionPHIDs(array $phids) {
$this->previousTransactionPHIDs = $phids;
return $this;
}
public function withFulltext($fulltext) {
$this->fulltext = $fulltext;
return $this;
}
public function execute() {
$table = new ConpherenceIndex();
$conn_r = $table->establishConnection('r');
$rows = queryfx_all(
$conn_r,
'SELECT threadPHID, transactionPHID, previousTransactionPHID
FROM %T i %Q %Q %Q',
$table->getTableName(),
$this->buildWhereClause($conn_r),
$this->buildOrderByClause($conn_r),
$this->buildLimitClause($conn_r));
return $rows;
}
protected function buildWhereClause(AphrontDatabaseConnection $conn_r) {
$where = array();
if ($this->threadPHIDs !== null) {
$where[] = qsprintf(
$conn_r,
'i.threadPHID IN (%Ls)',
$this->threadPHIDs);
}
if ($this->previousTransactionPHIDs !== null) {
$where[] = qsprintf(
$conn_r,
'i.previousTransactionPHID IN (%Ls)',
$this->previousTransactionPHIDs);
}
if (strlen($this->fulltext)) {
- $compiled_query = PhabricatorSearchDocument::newQueryCompiler()
- ->setQuery($this->fulltext)
- ->compileQuery();
+ $compiler = PhabricatorSearchDocument::newQueryCompiler();
+ $tokens = $compiler->newTokens($this->fulltext);
+ $compiled_query = $compiler->compileQuery($tokens);
$where[] = qsprintf(
$conn_r,
'MATCH(i.corpus) AGAINST (%s IN BOOLEAN MODE)',
$compiled_query);
}
return $this->formatWhereClause($where);
}
private function buildOrderByClause(AphrontDatabaseConnection $conn_r) {
if (strlen($this->fulltext)) {
return qsprintf(
$conn_r,
'ORDER BY MATCH(i.corpus) AGAINST (%s IN BOOLEAN MODE) DESC',
$this->fulltext);
} else {
return qsprintf(
$conn_r,
'ORDER BY id DESC');
}
}
}
diff --git a/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php b/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php
index 72c49576f0..626c7435c3 100644
--- a/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php
+++ b/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php
@@ -1,419 +1,420 @@
<?php
final class PhabricatorMySQLFulltextStorageEngine
extends PhabricatorFulltextStorageEngine {
public function getEngineIdentifier() {
return 'mysql';
}
public function getHostType() {
return new PhabricatorMySQLSearchHost($this);
}
public function reindexAbstractDocument(
PhabricatorSearchAbstractDocument $doc) {
$phid = $doc->getPHID();
if (!$phid) {
throw new Exception(pht('Document has no PHID!'));
}
$store = new PhabricatorSearchDocument();
$store->setPHID($doc->getPHID());
$store->setDocumentType($doc->getDocumentType());
$store->setDocumentTitle($doc->getDocumentTitle());
$store->setDocumentCreated($doc->getDocumentCreated());
$store->setDocumentModified($doc->getDocumentModified());
$store->replace();
$conn_w = $store->establishConnection('w');
$stemmer = new PhutilSearchStemmer();
$field_dao = new PhabricatorSearchDocumentField();
queryfx(
$conn_w,
'DELETE FROM %T WHERE phid = %s',
$field_dao->getTableName(),
$phid);
foreach ($doc->getFieldData() as $field) {
list($ftype, $corpus, $aux_phid) = $field;
$stemmed_corpus = $stemmer->stemCorpus($corpus);
queryfx(
$conn_w,
'INSERT INTO %T
(phid, phidType, field, auxPHID, corpus, stemmedCorpus) '.
'VALUES (%s, %s, %s, %ns, %s, %s)',
$field_dao->getTableName(),
$phid,
$doc->getDocumentType(),
$ftype,
$aux_phid,
$corpus,
$stemmed_corpus);
}
$sql = array();
foreach ($doc->getRelationshipData() as $relationship) {
list($rtype, $to_phid, $to_type, $time) = $relationship;
$sql[] = qsprintf(
$conn_w,
'(%s, %s, %s, %s, %d)',
$phid,
$to_phid,
$rtype,
$to_type,
$time);
}
$rship_dao = new PhabricatorSearchDocumentRelationship();
queryfx(
$conn_w,
'DELETE FROM %T WHERE phid = %s',
$rship_dao->getTableName(),
$phid);
if ($sql) {
queryfx(
$conn_w,
'INSERT INTO %T '.
'(phid, relatedPHID, relation, relatedType, relatedTime) '.
'VALUES %Q',
$rship_dao->getTableName(),
implode(', ', $sql));
}
}
/**
* Rebuild the PhabricatorSearchAbstractDocument that was used to index
* an object out of the index itself. This is primarily useful for debugging,
* as it allows you to inspect the search index representation of a
* document.
*
* @param phid PHID of a document which exists in the search index.
* @return null|PhabricatorSearchAbstractDocument Abstract document object
* which corresponds to the original abstract document used to
* build the document index.
*/
public function reconstructDocument($phid) {
$dao_doc = new PhabricatorSearchDocument();
$dao_field = new PhabricatorSearchDocumentField();
$dao_relationship = new PhabricatorSearchDocumentRelationship();
$t_doc = $dao_doc->getTableName();
$t_field = $dao_field->getTableName();
$t_relationship = $dao_relationship->getTableName();
$doc = queryfx_one(
$dao_doc->establishConnection('r'),
'SELECT * FROM %T WHERE phid = %s',
$t_doc,
$phid);
if (!$doc) {
return null;
}
$fields = queryfx_all(
$dao_field->establishConnection('r'),
'SELECT * FROM %T WHERE phid = %s',
$t_field,
$phid);
$relationships = queryfx_all(
$dao_relationship->establishConnection('r'),
'SELECT * FROM %T WHERE phid = %s',
$t_relationship,
$phid);
$adoc = id(new PhabricatorSearchAbstractDocument())
->setPHID($phid)
->setDocumentType($doc['documentType'])
->setDocumentTitle($doc['documentTitle'])
->setDocumentCreated($doc['documentCreated'])
->setDocumentModified($doc['documentModified']);
foreach ($fields as $field) {
$adoc->addField(
$field['field'],
$field['corpus'],
$field['auxPHID']);
}
foreach ($relationships as $relationship) {
$adoc->addRelationship(
$relationship['relation'],
$relationship['relatedPHID'],
$relationship['relatedType'],
$relationship['relatedTime']);
}
return $adoc;
}
public function executeSearch(PhabricatorSavedQuery $query) {
$table = new PhabricatorSearchDocument();
$document_table = $table->getTableName();
$conn = $table->establishConnection('r');
$subquery = $this->newFulltextSubquery($query, $conn);
$offset = (int)$query->getParameter('offset', 0);
$limit = (int)$query->getParameter('limit', 25);
// NOTE: We must JOIN the subquery in order to apply a limit.
$results = queryfx_all(
$conn,
'SELECT
documentPHID,
MAX(fieldScore) AS documentScore
FROM (%Q) query
JOIN %T root ON query.documentPHID = root.phid
GROUP BY documentPHID
ORDER BY documentScore DESC
LIMIT %d, %d',
$subquery,
$document_table,
$offset,
$limit);
return ipull($results, 'documentPHID');
}
private function newFulltextSubquery(
PhabricatorSavedQuery $query,
AphrontDatabaseConnection $conn) {
$field = new PhabricatorSearchDocumentField();
$field_table = $field->getTableName();
$document = new PhabricatorSearchDocument();
$document_table = $document->getTableName();
$select = array();
$select[] = 'document.phid AS documentPHID';
$join = array();
$where = array();
$title_field = PhabricatorSearchDocumentFieldType::FIELD_TITLE;
$title_boost = 1024;
$raw_query = $query->getParameter('query');
$compiled_query = $this->compileQuery($raw_query);
if (strlen($compiled_query)) {
$select[] = qsprintf(
$conn,
'IF(field.field = %s, %d, 0) +
MATCH(corpus, stemmedCorpus) AGAINST (%s IN BOOLEAN MODE)
AS fieldScore',
$title_field,
$title_boost,
$compiled_query);
$join[] = qsprintf(
$conn,
'%T field ON field.phid = document.phid',
$field_table);
$where[] = qsprintf(
$conn,
'MATCH(corpus, stemmedCorpus) AGAINST (%s IN BOOLEAN MODE)',
$compiled_query);
if ($query->getParameter('field')) {
$where[] = qsprintf(
$conn,
'field.field = %s',
$field);
}
} else {
$select[] = qsprintf(
$conn,
'document.documentCreated AS fieldScore');
}
$exclude = $query->getParameter('exclude');
if ($exclude) {
$where[] = qsprintf(
$conn,
'document.phid != %s',
$exclude);
}
$types = $query->getParameter('types');
if ($types) {
if (strlen($compiled_query)) {
$where[] = qsprintf(
$conn,
'field.phidType IN (%Ls)',
$types);
}
$where[] = qsprintf(
$conn,
'document.documentType IN (%Ls)',
$types);
}
$join[] = $this->joinRelationship(
$conn,
$query,
'authorPHIDs',
PhabricatorSearchRelationship::RELATIONSHIP_AUTHOR);
$statuses = $query->getParameter('statuses', array());
$statuses = array_fuse($statuses);
$open_rel = PhabricatorSearchRelationship::RELATIONSHIP_OPEN;
$closed_rel = PhabricatorSearchRelationship::RELATIONSHIP_CLOSED;
$include_open = !empty($statuses[$open_rel]);
$include_closed = !empty($statuses[$closed_rel]);
if ($include_open && !$include_closed) {
$join[] = $this->joinRelationship(
$conn,
$query,
'statuses',
$open_rel,
true);
} else if ($include_closed && !$include_open) {
$join[] = $this->joinRelationship(
$conn,
$query,
'statuses',
$closed_rel,
true);
}
if ($query->getParameter('withAnyOwner')) {
$join[] = $this->joinRelationship(
$conn,
$query,
'withAnyOwner',
PhabricatorSearchRelationship::RELATIONSHIP_OWNER,
true);
} else if ($query->getParameter('withUnowned')) {
$join[] = $this->joinRelationship(
$conn,
$query,
'withUnowned',
PhabricatorSearchRelationship::RELATIONSHIP_UNOWNED,
true);
} else {
$join[] = $this->joinRelationship(
$conn,
$query,
'ownerPHIDs',
PhabricatorSearchRelationship::RELATIONSHIP_OWNER);
}
$join[] = $this->joinRelationship(
$conn,
$query,
'subscriberPHIDs',
PhabricatorSearchRelationship::RELATIONSHIP_SUBSCRIBER);
$join[] = $this->joinRelationship(
$conn,
$query,
'projectPHIDs',
PhabricatorSearchRelationship::RELATIONSHIP_PROJECT);
$join[] = $this->joinRelationship(
$conn,
$query,
'repository',
PhabricatorSearchRelationship::RELATIONSHIP_REPOSITORY);
$select = implode(', ', $select);
$join = array_filter($join);
foreach ($join as $key => $clause) {
$join[$key] = ' JOIN '.$clause;
}
$join = implode(' ', $join);
if ($where) {
$where = 'WHERE '.implode(' AND ', $where);
} else {
$where = '';
}
if (strlen($compiled_query)) {
$order = '';
} else {
// When not executing a query, order by document creation date. This
// is the default view in object browser dialogs, like "Close Duplicate".
$order = qsprintf(
$conn,
'ORDER BY document.documentCreated DESC');
}
return qsprintf(
$conn,
'SELECT %Q FROM %T document %Q %Q %Q LIMIT 1000',
$select,
$document_table,
$join,
$where,
$order);
}
protected function joinRelationship(
AphrontDatabaseConnection $conn,
PhabricatorSavedQuery $query,
$field,
$type,
$is_existence = false) {
$sql = qsprintf(
$conn,
'%T AS %C ON %C.phid = document.phid AND %C.relation = %s',
id(new PhabricatorSearchDocumentRelationship())->getTableName(),
$field,
$field,
$field,
$type);
if (!$is_existence) {
$phids = $query->getParameter($field, array());
if (!$phids) {
return null;
}
$sql .= qsprintf(
$conn,
' AND %C.relatedPHID in (%Ls)',
$field,
$phids);
}
return $sql;
}
private function compileQuery($raw_query) {
$stemmer = new PhutilSearchStemmer();
$compiler = PhabricatorSearchDocument::newQueryCompiler()
- ->setQuery($raw_query)
->setStemmer($stemmer);
+ $tokens = $compiler->newTokens($raw_query);
+
$queries = array();
- $queries[] = $compiler->compileLiteralQuery();
- $queries[] = $compiler->compileStemmedQuery();
+ $queries[] = $compiler->compileLiteralQuery($tokens);
+ $queries[] = $compiler->compileStemmedQuery($tokens);
return implode(' ', array_filter($queries));
}
public function indexExists() {
return true;
}
public function getIndexStats() {
return false;
}
}
diff --git a/src/infrastructure/cluster/search/PhabricatorSearchService.php b/src/infrastructure/cluster/search/PhabricatorSearchService.php
index a9ceb0e7e5..8e5b296132 100644
--- a/src/infrastructure/cluster/search/PhabricatorSearchService.php
+++ b/src/infrastructure/cluster/search/PhabricatorSearchService.php
@@ -1,264 +1,268 @@
<?php
class PhabricatorSearchService
extends Phobject {
const KEY_REFS = 'cluster.search.refs';
protected $config;
protected $disabled;
protected $engine;
protected $hosts = array();
protected $hostsConfig;
protected $hostType;
protected $roles = array();
const STATUS_OKAY = 'okay';
const STATUS_FAIL = 'fail';
const ROLE_WRITE = 'write';
const ROLE_READ = 'read';
public function __construct(PhabricatorFulltextStorageEngine $engine) {
$this->engine = $engine;
$this->hostType = $engine->getHostType();
}
/**
* @throws Exception
*/
public function newHost($config) {
$host = clone($this->hostType);
$host_config = $this->config + $config;
$host->setConfig($host_config);
$this->hosts[] = $host;
return $host;
}
public function getEngine() {
return $this->engine;
}
public function getDisplayName() {
return $this->hostType->getDisplayName();
}
public function getStatusViewColumns() {
return $this->hostType->getStatusViewColumns();
}
public function setConfig($config) {
$this->config = $config;
if (!isset($config['hosts'])) {
$config['hosts'] = array(
array(
'host' => idx($config, 'host'),
'port' => idx($config, 'port'),
'protocol' => idx($config, 'protocol'),
'roles' => idx($config, 'roles'),
),
);
}
foreach ($config['hosts'] as $host) {
$this->newHost($host);
}
}
public function getConfig() {
return $this->config;
}
public static function getConnectionStatusMap() {
return array(
self::STATUS_OKAY => array(
'icon' => 'fa-exchange',
'color' => 'green',
'label' => pht('Okay'),
),
self::STATUS_FAIL => array(
'icon' => 'fa-times',
'color' => 'red',
'label' => pht('Failed'),
),
);
}
public function isWritable() {
return (bool)$this->getAllHostsForRole(self::ROLE_WRITE);
}
public function isReadable() {
return (bool)$this->getAllHostsForRole(self::ROLE_READ);
}
public function getPort() {
return idx($this->config, 'port');
}
public function getProtocol() {
return idx($this->config, 'protocol');
}
public function getVersion() {
return idx($this->config, 'version');
}
public function getHosts() {
return $this->hosts;
}
/**
* Get a random host reference with the specified role, skipping hosts which
* failed recent health checks.
* @throws PhabricatorClusterNoHostForRoleException if no healthy hosts match.
* @return PhabricatorSearchHost
*/
public function getAnyHostForRole($role) {
$hosts = $this->getAllHostsForRole($role);
shuffle($hosts);
foreach ($hosts as $host) {
$health = $host->getHealthRecord();
if ($health->getIsHealthy()) {
return $host;
}
}
throw new PhabricatorClusterNoHostForRoleException($role);
}
/**
* Get all configured hosts for this service which have the specified role.
* @return PhabricatorSearchHost[]
*/
public function getAllHostsForRole($role) {
// if the role is explicitly set to false at the top level, then all hosts
// have the role disabled.
if (idx($this->config, $role) === false) {
return array();
}
$hosts = array();
foreach ($this->hosts as $host) {
if ($host->hasRole($role)) {
$hosts[] = $host;
}
}
return $hosts;
}
/**
* Get a reference to all configured fulltext search cluster services
* @return PhabricatorSearchService[]
*/
public static function getAllServices() {
$cache = PhabricatorCaches::getRequestCache();
$refs = $cache->getKey(self::KEY_REFS);
if (!$refs) {
$refs = self::newRefs();
$cache->setKey(self::KEY_REFS, $refs);
}
return $refs;
}
/**
* Load all valid PhabricatorFulltextStorageEngine subclasses
*/
public static function loadAllFulltextStorageEngines() {
return id(new PhutilClassMapQuery())
->setAncestorClass('PhabricatorFulltextStorageEngine')
->setUniqueMethod('getEngineIdentifier')
->execute();
}
/**
* Create instances of PhabricatorSearchService based on configuration
* @return PhabricatorSearchService[]
*/
public static function newRefs() {
$services = PhabricatorEnv::getEnvConfig('cluster.search');
$engines = self::loadAllFulltextStorageEngines();
$refs = array();
foreach ($services as $config) {
// Normally, we've validated configuration before we get this far, but
// make sure we don't fatal if we end up here with a bogus configuration.
if (!isset($engines[$config['type']])) {
throw new Exception(
pht(
'Configured search engine type "%s" is unknown. Valid engines '.
'are: %s.',
$config['type'],
implode(', ', array_keys($engines))));
}
$engine = clone($engines[$config['type']]);
$cluster = new self($engine);
$cluster->setConfig($config);
$engine->setService($cluster);
$refs[] = $cluster;
}
return $refs;
}
/**
* (re)index the document: attempt to pass the document to all writable
* fulltext search hosts
*/
public static function reindexAbstractDocument(
PhabricatorSearchAbstractDocument $document) {
$exceptions = array();
foreach (self::getAllServices() as $service) {
if (!$service->isWritable()) {
continue;
}
$engine = $service->getEngine();
try {
$engine->reindexAbstractDocument($document);
} catch (Exception $ex) {
$exceptions[] = $ex;
}
}
if ($exceptions) {
throw new PhutilAggregateException(
pht(
'Writes to search services failed while reindexing document "%s".',
$document->getPHID()),
$exceptions);
}
}
/**
* Execute a full-text query and return a list of PHIDs of matching objects.
* @return string[]
* @throws PhutilAggregateException
*/
public static function executeSearch(PhabricatorSavedQuery $query) {
$exceptions = array();
// try all services until one succeeds
foreach (self::getAllServices() as $service) {
try {
$engine = $service->getEngine();
$res = $engine->executeSearch($query);
// return immediately if we get results
return $res;
+ } catch (PhutilSearchQueryCompilerSyntaxException $ex) {
+ // If there's a query compilation error, return it directly to the
+ // user: they issued a query with bad syntax.
+ throw $ex;
} catch (Exception $ex) {
$exceptions[] = $ex;
}
}
$msg = pht('All of the configured Fulltext Search services failed.');
throw new PhutilAggregateException($msg, $exceptions);
}
}

File Metadata

Mime Type
text/x-diff
Expires
Fri, Mar 14, 7:30 AM (3 h, 47 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
71681
Default Alt Text
(21 KB)

Event Timeline

styx copyright 2020 sirocyl / ellie (kymkdd) / ncommander / styx contributors
Use of other names and brands is not intended to reflect a claim of, or right to, use the name or brand.
styx is an operating system using the Linux kernel and components, and "styx Linux" as a phrase does not reflect the inclusion of the Linux brand or trademark in the styx project.
Linux® is the registered trademark of Linus Torvalds in the U.S. and other countries.
Terms and Conditions (TODO: fix this!)