[TikiWiki-commits] [Git][tikiwiki/tiki][26.x] 4 commits: [ENH] manticore/mysql index optimization: allow queries to specify only a...

Victor Emanouilov pushed to branch 26.x at Tiki Wiki CMS Groupware / Tiki


Commits:
8453dba2 by Victor Emanouilov at 2025-09-22T12:59:31+03:00
[ENH] manticore/mysql index optimization: allow queries to specify only a subset of the fields to be returned; use this in objectlib getFormattedTitle (used extensively by Relation field) and search lookup formatted results

- - - - -
252fcf1f by Victor Emanouilov at 2025-09-22T13:00:38+03:00
[FIX] phpcs ignore list fix

- - - - -
48254db0 by Victor Emanouilov at 2025-09-22T13:00:41+03:00
[ENH] ability to select fields from list context to reduce memory usage - especially useful for large indexes where calc/subtotal needs to run over many results

- - - - -
9b574bed by Victor Emanouilov at 2025-09-22T13:15:57+03:00
[FIX] missing isMainField method in tracker AbstractField

- - - - -


9 changed files:

- doc/devtools/codesniffer/standards/TikiIgnore/ignore_list.json
- lib/core/Search/Elastic/Index.php
- lib/core/Search/Manticore/Index.php
- lib/core/Search/MySql/Index.php
- lib/core/Search/Query.php
- lib/core/Search/Query/WikiBuilder.php
- lib/core/Services/Search/Controller.php
- lib/core/Tracker/Field/AbstractField.php
- lib/objectlib.php


Changes:

=====================================
doc/devtools/codesniffer/standards/TikiIgnore/ignore_list.json
=====================================
@@ -4998,6 +4998,7 @@
             "Search_Query_WikiBuilder::wpquery_pagination_sort_jsvar": true,
             "Search_Query_WikiBuilder::wpquery_pagination_sort_arg": true,
             "Search_Query_WikiBuilder::wpquery_pagination_max": true,
+            "Search_Query_WikiBuilder::wpquery_select_fields": true,
             "Search_Query_WikiBuilder::wpquery_group_boost": true,
             "Search_Query_WikiBuilder::wpquery_index_federated": true,
             "Search_Query_WikiBuilder::get_fields_from_arguments": true


=====================================
lib/core/Search/Elastic/Index.php
=====================================
@@ -404,6 +404,14 @@ class Search_Elastic_Index implements Search_Index_Interface, Search_Index_Query
                 ]];
             }
 
+            if ($selectFields = $query->getSelectionFields()) {
+                $sourcePart = [
+                    "_source" => $selectFields,
+                ];
+            } else {
+                $sourcePart = [];
+            }
+
             $indices = [$this->index];
 
             $foreign = array_map(function ($query) use ($builder) {
@@ -454,6 +462,7 @@ class Search_Elastic_Index implements Search_Index_Interface, Search_Index_Query
                 $facetPart,
                 $rescorePart,
                 $postFilterPart,
+                $sourcePart,
                 [
                     "from" => $resultStart,
                     "size" => $resultCount,


=====================================
lib/core/Search/Manticore/Index.php
=====================================
@@ -396,7 +396,15 @@ class Index implements \Search_Index_Interface, \Search_Index_QueryRepository
         $builder->setPossibleFields($this->pdo_client->possibleFacetFields($table));
         $facets = $builder->build($query->getFacets());
 
-        $sql = "SELECT *";
+        if ($selectionFields = $query->getSelectionFields()) {
+            foreach ($selectionFields as $key => $field) {
+                $this->ensureHasField($field);
+                $selectionFields[$key] = strtolower($field);
+            }
+            $sql = "SELECT " . implode(',', $selectionFields);
+        } else {
+            $sql = "SELECT *";
+        }
 
         foreach ($select as $key => $expr) {
             $sql .= ", $expr as $key";


=====================================
lib/core/Search/MySql/Index.php
=====================================
@@ -132,7 +132,13 @@ class Search_MySql_Index implements Search_Index_Interface
 
             $order = $this->getOrderClause($query, (bool) $scoreFields);
 
-            $selectFields = $this->table->all();
+            if ($selectFields = $query->getSelectionFields()) {
+                $selectFields = array_map(function($field) {
+                    return $this->tfTranslator->shortenize($field);
+                }, $selectFields);
+            } else  {
+                $selectFields = $this->table->all();
+            }
 
             if ($scoreFields) {
                 $str = $this->db->qstr(implode(' ', $words));


=====================================
lib/core/Search/Query.php
=====================================
@@ -13,6 +13,7 @@ class Search_Query implements Search_Query_Interface
     private $count = 50;
     private $weightCalculator = null;
     private $identifierFields = null;
+    private $selectionFields = null;
 
     private $postFilter;
     private $subQueries = [];
@@ -44,6 +45,16 @@ class Search_Query implements Search_Query_Interface
         $this->identifierFields = $fields;
     }
 
+    public function setSelectionFields(array $fields)
+    {
+        $this->selectionFields = $fields;
+    }
+
+    public function getSelectionFields()
+    {
+        return $this->selectionFields;
+    }
+
     public function addObject($type, $objectId)
     {
         if (is_null($this->objectList)) {


=====================================
lib/core/Search/Query/WikiBuilder.php
=====================================
@@ -123,6 +123,11 @@ class Search_Query_WikiBuilder
         $this->paginationArguments['max'] = max(1, (int) $value);
     }
 
+    public function wpquery_select_fields($query, $value)
+    {
+        $query->setSelectionFields(preg_split('/[, ]+/', $value));
+    }
+
     public function wpquery_filter_editable($query, $editableType, array $arguments)
     {
         $fields = $this->get_fields_from_arguments($arguments);


=====================================
lib/core/Services/Search/Controller.php
=====================================
@@ -185,7 +185,9 @@ class Services_Search_Controller
             /** @var UnifiedSearchLib $lib */
             $lib = TikiLib::lib('unifiedsearch');
 
-            if (! empty($filter['title']) && preg_match_all('/\{(\w+)\}/', $format, $matches)) {
+            $format_pattern = '/\{([\w\.]+)\}/';
+
+            if (! empty($filter['title']) && preg_match_all($format_pattern, $format, $matches)) {
                 // formatted object_selector search results should also search in formatted fields besides the title
                 $titleFilter = $filter['title'];
                 unset($filter['title']);
@@ -213,14 +215,18 @@ class Services_Search_Controller
             $query->setOrder($input->sort_order->text() ?: 'title_asc');
             $query->setRange($input->offset->int(), $input->maxRecords->int() ?: $prefs['maxRecords']);
 
+            if (preg_match_all($format_pattern, $format, $m)) {
+                $query->setSelectionFields($m[1]);
+            }
+
             $result = $query->search($lib->getIndex());
 
-            $result->applyTransform(function ($item) use ($format, $smarty, $titleFilter, $highlightHelper) {
+            $result->applyTransform(function ($item) use ($format, $format_pattern, $smarty, $titleFilter, $highlightHelper) {
                 $transformed = [
                     'object_type' => $item['object_type'],
                     'object_id' => $item['object_id'],
                     'parent_id' => $item['gallery_id'],
-                    'title' => preg_replace_callback('/\{([\w\.]+)\}/', function ($matches) use ($item, $format, $titleFilter, $highlightHelper) {
+                    'title' => preg_replace_callback($format_pattern, function ($matches) use ($item, $format, $titleFilter, $highlightHelper) {
                         $key = $matches[1];
                         if (isset($item[$key])) {
                             // if this is a trackeritem we do not want only the name but also the trackerid listed when setting up a field


=====================================
lib/core/Tracker/Field/AbstractField.php
=====================================
@@ -623,4 +623,9 @@ abstract class AbstractField implements FieldInterface, IndexableInterface
     {
         return '';
     }
+
+    public function isMainField(): bool
+    {
+        return ($this->getConfiguration('isMain') == 'y') ? true : false;
+    }
 }


=====================================
lib/objectlib.php
=====================================
@@ -786,9 +786,13 @@ class ObjectLib extends TikiLib
                 'object_type' => $type,
                 'object_id'   => $id
             ]);
+            $format_pattern = '/\{([\w\.]+)\}/';
+            if (preg_match_all($format_pattern, $format, $m)) {
+                $query->setSelectionFields($m[1]);
+            }
             $result = $query->search($lib->getIndex());
-            $result->applyTransform(function ($item) use ($format, $metadata) {
-                return preg_replace_callback('/\{([\w\.]+)\}/', function ($matches) use ($item, $format, $metadata) {
+            $result->applyTransform(function ($item) use ($format, $format_pattern, $metadata) {
+                return preg_replace_callback($format_pattern, function ($matches) use ($item, $format, $metadata) {
                     $key = $matches[1];
                     if (isset($item[$key])) {
                         return $item[$key];



View it on GitLab: https://gitlab.com/tikiwiki/tiki/-/compare/640d382a00cfc0734f0e400e83b10073750905de...9b574bed8aad7cfac43728f024a7725b2c72bf0a

-- 
View it on GitLab: https://gitlab.com/tikiwiki/tiki/-/compare/640d382a00cfc0734f0e400e83b10073750905de...9b574bed8aad7cfac43728f024a7725b2c72bf0a
You're receiving this email because of your account on gitlab.com.

_______________________________________________
TikiWiki-cvs mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/tikiwiki-cvs