Merge branch '1.3' into 1.x

Author: lukmzig

config/pimcore/config.yaml

@@ -17,56 +17,6 @@ pimcore_generic_data_index:
               type: text
               analyzer: standard
               search_analyzer: generic_data_index_whitespace_analyzer
-
-    index_settings:
-      index:
-        mapping:
-          nested_fields:
-            limit: 200
-          total_fields:
-            limit: 100000
-      number_of_shards: 1
-      number_of_replicas: 0
-      max_ngram_diff: 30
-      max_result_window: 10000
-      analysis:
-        analyzer:
-          generic_data_index_ngram_analyzer:
-            tokenizer: generic_data_index_ngram_tokenzier
-            filter:
-              - lowercase
-          generic_data_index_whitespace_analyzer:
-            tokenizer: generic_data_index_whitespace_tokenzier
-            filter:
-              - lowercase
-          generic_data_index_path_analyzer:
-            tokenizer: generic_data_index_path_tokenizer
-        normalizer:
-          generic_data_index_sort_normalizer:
-            type: custom
-            filter:
-              - lowercase
-          generic_data_index_sort_truncate_normalizer:
-            type: custom
-            char_filter:
-              - generic_data_index_sort_truncate
-            filter:
-              - lowercase
-        tokenizer:
-          generic_data_index_ngram_tokenzier:
-            type: ngram
-            min_gram: 3
-            max_gram: 25
-            token_chars: [ letter, digit ]
-          generic_data_index_whitespace_tokenzier:
-            type: whitespace
-          generic_data_index_path_tokenizer:
-            type: "path_hierarchy"
-        char_filter:
-          generic_data_index_sort_truncate:
-            type: pattern_replace
-            pattern: ^(.{256})(.*)$
-            replacement: $1
     system_fields_settings:
       general:
         id:

config/pimcore/default_index_settings.yaml

@@ -0,0 +1,48 @@
+index:
+  mapping:
+    nested_fields:
+      limit: 200
+    total_fields:
+      limit: 100000
+number_of_shards: 1
+number_of_replicas: 0
+max_ngram_diff: 30
+max_result_window: 10000
+analysis:
+  analyzer:
+    generic_data_index_ngram_analyzer:
+      tokenizer: generic_data_index_ngram_tokenzier
+      filter:
+        - lowercase
+    generic_data_index_whitespace_analyzer:
+      tokenizer: generic_data_index_whitespace_tokenzier
+      filter:
+        - lowercase
+    generic_data_index_path_analyzer:
+      tokenizer: generic_data_index_path_tokenizer
+  normalizer:
+    generic_data_index_sort_normalizer:
+      type: custom
+      filter:
+        - lowercase
+    generic_data_index_sort_truncate_normalizer:
+      type: custom
+      char_filter:
+        - generic_data_index_sort_truncate
+      filter:
+        - lowercase
+  tokenizer:
+    generic_data_index_ngram_tokenzier:
+      type: ngram
+      min_gram: 3
+      max_gram: 25
+      token_chars: [ letter, digit ]
+    generic_data_index_whitespace_tokenzier:
+      type: whitespace
+    generic_data_index_path_tokenizer:
+      type: "path_hierarchy"
+  char_filter:
+    generic_data_index_sort_truncate:
+      type: pattern_replace
+      pattern: ^(.{256})(.*)$
+      replacement: $1

doc/02_Configuration/03_Index_Management.md

@@ -71,21 +71,26 @@ pimcore_generic_data_index:
 
 ### Repairing Indices
 
-Sometimes it might be needed to delete and recreate the index (for example if the mapping changed and
-cannot be updated).
+Sometimes it might be needed to delete and recreate the index from the Pimcore database
+(for example if the mapping changed and cannot be updated).
 
-Do this with the index update command and pass -r option.
+Do this with the index update command and pass `-r` option (which deletes and recreates the index).
 ```
 # delete index and recreate it
 bin/console generic-data-index:update:index -r
 ```
+Without the `-r` option, the index mapping is just updated and all items are added into the queue
+for a reindex from the Pimcore database. 
+
 
 ### Updating Index Structure for Data Indices
 
 Index mapping is updated automatically e.g. when adding system languages or new fields to the class definition. 
 Sometimes it might be necessary to update the index structure manually.
 
-Do this with the reindex command.
+Do this with the reindex command. This command does native opensearch/elasticsearch re-indexing. So it does not 
+index data from the database but reindexes data within the search indices.
+
 ```
 # updates index mapping with native reindexing
 bin/console generic-data-index:reindex
@@ -130,4 +135,4 @@ After every class definition update you should run the following command to upda
 php bin/console generic-data-index:deployment:reindex
 ```
 
-This command will update the index structure for all data object classes which were created/updated since the last deployment and reindex all data objects for relevant classes.
+This command will update the index structure for all data object classes which were created/updated since the last deployment and reindex all data objects for relevant classes.

src/DependencyInjection/PimcoreGenericDataIndexExtension.php

@@ -71,19 +71,14 @@ public function prepend(ContainerBuilder $container): void
             $loader->load('doctrine_migrations.yaml');
         }
 
-        $filename = __DIR__ . '/../../config/doctrine.yaml';
-
-        try {
-            $config = Yaml::parseFile($filename);
-        } catch (ParseException $e) {
-            throw new InvalidArgumentException(sprintf('The file "%s" does not contain valid YAML.', $filename), 0, $e);
-        }
+        $config = $this->getParsedConfig(__DIR__ . '/../../config/doctrine.yaml');
 
         $container->prependExtensionConfig('doctrine', $config['doctrine']);
     }
 
     private function registerIndexServiceParams(ContainerBuilder $container, array $indexSettings): void
     {
+        $indexSettings['index_settings'] = $this->getIndexSettings($indexSettings['index_settings']);
         $definition = $container->getDefinition(SearchIndexConfigServiceInterface::class);
         $definition->setArgument('$clientType', $indexSettings['client_params']['client_type']);
         $definition->setArgument('$indexPrefix', $indexSettings['client_params']['index_prefix']);
@@ -109,6 +104,42 @@ private function registerIndexServiceParams(ContainerBuilder $container, array $
         $definition->setArgument('$queueSettings', $indexSettings['queue_settings']);
     }
 
+    private function getIndexSettings(array $indexSettings): array
+    {
+        $defaultIndexSettings = $this->getParsedConfig(
+            __DIR__ . '/../../config/pimcore/default_index_settings.yaml'
+        );
+
+        return $this->mergeConfig($defaultIndexSettings, $indexSettings);
+    }
+
+    private function mergeConfig(array $default, array $custom): array
+    {
+        foreach ($custom as $key => $value) {
+            $default[$key] = is_array($value) && isset($default[$key]) && is_array($default[$key])
+                ? $this->mergeConfig($default[$key], $value)
+                : $value;
+        }
+
+        return $default;
+    }
+
+    /**
+     * @throws InvalidArgumentException
+     */
+    private function getParsedConfig(string $fileLocation): array
+    {
+        try {
+            return Yaml::parseFile($fileLocation);
+        } catch (ParseException $e) {
+            throw new InvalidArgumentException(
+                sprintf('The file "%s" does not contain valid YAML.', $fileLocation),
+                0,
+                $e
+            );
+        }
+    }
+
     /**
      * @throws InvalidArgumentException
      */