Page MenuHomestyx hydra
Files F1454921

No OneTemporary
Actions

View Options

diff --git a/resources/sql/autopatches/20170830.ferret.01.unique.sql b/resources/sql/autopatches/20170830.ferret.01.unique.sql
new file mode 100644
index 0000000000..f76c5050e8
--- /dev/null
+++ b/resources/sql/autopatches/20170830.ferret.01.unique.sql
@@ -0,0 +1,4 @@
+TRUNCATE TABLE {$NAMESPACE}_maniphest.maniphest_task_ffield;
+
+ALTER TABLE {$NAMESPACE}_maniphest.maniphest_task_ffield
+ ADD UNIQUE KEY `key_documentfield` (documentID, fieldKey);
diff --git a/src/applications/search/constants/PhabricatorSearchDocumentFieldType.php b/src/applications/search/constants/PhabricatorSearchDocumentFieldType.php
index 10dbf0ca65..4dd49e8c92 100644
--- a/src/applications/search/constants/PhabricatorSearchDocumentFieldType.php
+++ b/src/applications/search/constants/PhabricatorSearchDocumentFieldType.php
@@ -1,9 +1,10 @@
<?php
final class PhabricatorSearchDocumentFieldType extends Phobject {
const FIELD_TITLE = 'titl';
const FIELD_BODY = 'body';
const FIELD_COMMENT = 'cmnt';
+ const FIELD_ALL = 'full';
}
diff --git a/src/applications/search/engineextension/PhabricatorFerretFulltextEngineExtension.php b/src/applications/search/engineextension/PhabricatorFerretFulltextEngineExtension.php
index 04d2fad608..bafeca2c81 100644
--- a/src/applications/search/engineextension/PhabricatorFerretFulltextEngineExtension.php
+++ b/src/applications/search/engineextension/PhabricatorFerretFulltextEngineExtension.php
@@ -1,133 +1,160 @@
<?php
final class PhabricatorFerretFulltextEngineExtension
extends PhabricatorFulltextEngineExtension {
const EXTENSIONKEY = 'ferret';
public function getExtensionName() {
return pht('Ferret Fulltext Engine');
}
public function shouldIndexFulltextObject($object) {
return ($object instanceof PhabricatorFerretInterface);
}
public function indexFulltextObject(
$object,
PhabricatorSearchAbstractDocument $document) {
$phid = $document->getPHID();
$engine = $object->newFerretEngine();
$ferret_document = $engine->newDocumentObject()
->setObjectPHID($phid)
->setIsClosed(0)
->setEpochCreated(0)
->setEpochModified(0);
$stemmer = new PhutilSearchStemmer();
- $ferret_fields = array();
- $ngrams_source = array();
+ $key_all = PhabricatorSearchDocumentFieldType::FIELD_ALL;
+
+ $empty_template = array(
+ 'raw' => array(),
+ 'normal' => array(),
+ );
+
+ $ferret_corpus_map = array(
+ $key_all => $empty_template,
+ );
+
foreach ($document->getFieldData() as $field) {
list($key, $raw_corpus) = $field;
-
if (!strlen($raw_corpus)) {
continue;
}
$normal_corpus = $stemmer->stemCorpus($raw_corpus);
+ if (!isset($ferret_corpus_map[$key])) {
+ $ferret_corpus_map[$key] = $empty_template;
+ }
+
+ $ferret_corpus_map[$key]['raw'][] = $raw_corpus;
+ $ferret_corpus_map[$key]['normal'][] = $normal_corpus;
+
+ $ferret_corpus_map[$key_all]['raw'][] = $raw_corpus;
+ $ferret_corpus_map[$key_all]['normal'][] = $normal_corpus;
+ }
+
+ $ferret_fields = array();
+ foreach ($ferret_corpus_map as $key => $fields) {
+ $raw_corpus = $fields['raw'];
+ $raw_corpus = implode("\n", $raw_corpus);
+
+ $normal_corpus = $fields['normal'];
+ $normal_corpus = implode("\n", $normal_corpus);
+
$ferret_fields[] = $engine->newFieldObject()
->setFieldKey($key)
->setRawCorpus($raw_corpus)
->setNormalCorpus($normal_corpus);
-
- $ngrams_source[] = $raw_corpus;
}
- $ngrams_source = implode(' ', $ngrams_source);
+
+ $ngrams_source = $ferret_corpus_map[$key_all]['raw'];
+ $ngrams_source = implode("\n", $ngrams_source);
$ngrams = id(new PhabricatorNgramEngine())
->getNgramsFromString($ngrams_source, 'index');
$ferret_document->openTransaction();
try {
$this->deleteOldDocument($engine, $object, $document);
$ferret_document->save();
$document_id = $ferret_document->getID();
foreach ($ferret_fields as $ferret_field) {
$ferret_field
->setDocumentID($document_id)
->save();
}
$ferret_ngrams = $engine->newNgramsObject();
$conn = $ferret_ngrams->establishConnection('w');
$sql = array();
foreach ($ngrams as $ngram) {
$sql[] = qsprintf(
$conn,
'(%d, %s)',
$document_id,
$ngram);
}
foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) {
queryfx(
$conn,
'INSERT INTO %T (documentID, ngram) VALUES %Q',
$ferret_ngrams->getTableName(),
$chunk);
}
} catch (Exception $ex) {
$ferret_document->killTransaction();
throw $ex;
}
$ferret_document->saveTransaction();
}
private function deleteOldDocument(
PhabricatorFerretEngine $engine,
$object,
PhabricatorSearchAbstractDocument $document) {
$old_document = $engine->newDocumentObject()->loadOneWhere(
'objectPHID = %s',
$document->getPHID());
if (!$old_document) {
return;
}
$conn = $old_document->establishConnection('w');
$old_id = $old_document->getID();
queryfx(
$conn,
'DELETE FROM %T WHERE id = %d',
$engine->newDocumentObject()->getTableName(),
$old_id);
queryfx(
$conn,
'DELETE FROM %T WHERE documentID = %d',
$engine->newFieldObject()->getTableName(),
$old_id);
queryfx(
$conn,
'DELETE FROM %T WHERE documentID = %d',
$engine->newNgramsObject()->getTableName(),
$old_id);
}
}
diff --git a/src/applications/search/ferret/PhabricatorFerretField.php b/src/applications/search/ferret/PhabricatorFerretField.php
index 5b2370ae8f..cd7e7c68d5 100644
--- a/src/applications/search/ferret/PhabricatorFerretField.php
+++ b/src/applications/search/ferret/PhabricatorFerretField.php
@@ -1,36 +1,37 @@
<?php
abstract class PhabricatorFerretField
extends PhabricatorSearchDAO {
protected $documentID;
protected $fieldKey;
protected $rawCorpus;
protected $normalCorpus;
abstract public function getIndexKey();
protected function getConfiguration() {
return array(
self::CONFIG_TIMESTAMPS => false,
self::CONFIG_COLUMN_SCHEMA => array(
'documentID' => 'uint32',
'fieldKey' => 'text4',
'rawCorpus' => 'sort',
'normalCorpus' => 'sort',
),
self::CONFIG_KEY_SCHEMA => array(
- 'key_document' => array(
+ 'key_documentfield' => array(
'columns' => array('documentID', 'fieldKey'),
+ 'unique' => true,
),
),
) + parent::getConfiguration();
}
public function getTableName() {
$application = $this->getApplicationName();
$key = $this->getIndexKey();
return "{$application}_{$key}_ffield";
}
}

File Metadata

Mime Type
text/x-diff
Expires
Thu, Nov 13, 4:16 PM (2 h, 3 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
336300
Default Alt Text
(6 KB)

Event Timeline

styx copyright 2020 sirocyl / ellie (kymkdd) / ncommander / styx contributors
Use of other names and brands is not intended to reflect a claim of, or right to, use the name or brand.
styx is an operating system using the Linux kernel and components, and "styx Linux" as a phrase does not reflect the inclusion of the Linux brand or trademark in the styx project.
Linux® is the registered trademark of Linus Torvalds in the U.S. and other countries.
Terms and Conditions (TODO: fix this!)