[Improvement] Add functionality to enqueue dependent items

config/services/search/search-services.yaml

@@ -40,6 +40,9 @@ services:
   Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchResultItem\LazyLoading\AssetLazyLoadingHandlerInterface:
     class: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchResultItem\LazyLoading\AssetLazyLoadingHandler
 
+  Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\RequiredByElementListServiceInterface:
+    class: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\RequiredByElementListService
+
   Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchResultItem\LazyLoading\DataObjectLazyLoadingHandlerInterface:
     class: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchResultItem\LazyLoading\DataObjectLazyLoadingHandler
 

src/Model/SearchIndex/HitData.php

@@ -0,0 +1,42 @@
+<?php
+declare(strict_types=1);
+
+/**
+ * Pimcore
+ *
+ * This source file is available under two different licenses:
+ * - GNU General Public License version 3 (GPLv3)
+ * - Pimcore Commercial License (PCL)
+ * Full copyright and license information is available in
+ * LICENSE.md which is distributed with this source code.
+ *
+ *  @copyright  Copyright (c) Pimcore GmbH (http://www.pimcore.org)
+ *  @license    http://www.pimcore.org/license     GPLv3 and PCL
+ */
+
+namespace Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndex;
+
+final readonly class HitData
+{
+    public function __construct(
+        private string $id,
+        private string $elementType,
+        private string $index
+    ) {
+    }
+
+    public function getId(): string
+    {
+        return $this->id;
+    }
+
+    public function getElementType(): string
+    {
+        return $this->elementType;
+    }
+
+    public function getIndex(): string
+    {
+        return $this->index;
+    }
+}

src/Repository/IndexQueueRepository.php

@@ -17,13 +17,16 @@
 namespace Pimcore\Bundle\GenericDataIndexBundle\Repository;
 
 use Doctrine\DBAL\Connection;
+use Doctrine\DBAL\Exception as DBALException;
 use Doctrine\DBAL\Query\QueryBuilder as DBALQueryBuilder;
 use Doctrine\ORM\EntityManagerInterface;
 use Doctrine\ORM\NonUniqueResultException;
 use Doctrine\ORM\NoResultException;
 use Doctrine\ORM\QueryBuilder;
 use Exception;
 use Pimcore\Bundle\GenericDataIndexBundle\Entity\IndexQueue;
+use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\IndexQueueOperation;
+use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndex\HitData;
 use Pimcore\Bundle\GenericDataIndexBundle\Service\TimeServiceInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\Traits\LoggerAwareTrait;
 use Symfony\Component\Serializer\Exception\ExceptionInterface;
@@ -105,7 +108,7 @@ public function getUnhandledIndexQueueEntries(
     /**
      * @param IndexQueue[] $entries
      *
-     * @throws \Doctrine\DBAL\Exception
+     * @throws DBALException
      */
     public function deleteQueueEntries(array $entries): void
     {
@@ -166,12 +169,12 @@ public function generateSelectQuery(
     }
 
     /**
-     * @throws \Doctrine\DBAL\Exception
+     * @throws DBALException
      */
     public function enqueueBySelectQuery(DBALQueryBuilder $queryBuilder): void
     {
         $sql = <<<SQL
-            INSERT INTO 
+            INSERT INTO
                 %s (elementId, elementType, elementIndexName, operation, operationTime, dispatched)
                 %s
                 ON DUPLICATE KEY
@@ -186,7 +189,45 @@ public function enqueueBySelectQuery(DBALQueryBuilder $queryBuilder): void
     }
 
     /**
-     * @throws \Doctrine\DBAL\Exception
+     * @throws DBALException
+     *
+     * @param HitData[] $enqueueItemList
+     */
+    public function enqueueByItemList(array $enqueueItemList, IndexQueueOperation $operation, int $operationTime): void
+    {
+        if (empty($enqueueItemList)) {
+            return;
+        }
+
+        $sql = <<<SQL
+            INSERT INTO
+                %s (elementId, elementType, elementIndexName, operation, operationTime, dispatched)
+                VALUES %s
+                ON DUPLICATE KEY
+                UPDATE
+                    operation = VALUES(operation),
+                    operationTime = VALUES(operationTime),
+                    dispatched = VALUES(dispatched)
+        SQL;
+
+        $values = [];
+        foreach ($enqueueItemList as $item) {
+            $values[] = sprintf(
+                '(%s, %s, %s, %s, %s, 0)',
+                $this->connection->quote($item->getId()),
+                $this->connection->quote($item->getElementType()),
+                $this->connection->quote($item->getIndex()),
+                $this->connection->quote($operation->value),
+                $operationTime
+            );
+        }
+        $this->connection->executeQuery(
+            sprintf($sql, IndexQueue::TABLE, implode(',', $values))
+        );
+    }
+
+    /**
+     * @throws DBALException
      */
     public function dispatchItems(
         int $limit

src/SearchIndexAdapter/DefaultSearch/Asset/FieldDefinitionAdapter/AbstractAdapter.php

@@ -141,4 +141,4 @@ protected function throwInvalidFilterValueArgumentException(mixed $value, AssetM
             )
         );
     }
-}
+}

src/SearchIndexAdapter/DefaultSearch/DataObject/FieldDefinitionAdapter/AbstractAdapter.php

@@ -99,4 +99,4 @@ public function getInheritedData(
 
         return $this->getInheritedData($parent, $objectId, $value, $key, $language, $callback);
     }
-}
+}

src/SearchIndexAdapter/DefaultSearch/DefaultSearchService.php

@@ -21,6 +21,7 @@
 use Pimcore\Bundle\GenericDataIndexBundle\Exception\DefaultSearch\SearchFailedException;
 use Pimcore\Bundle\GenericDataIndexBundle\Exception\SwitchIndexAliasException;
 use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\Debug\SearchInformation;
+use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\DefaultSearchInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\Search;
 use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Interfaces\AdapterSearchInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndexAdapter\SearchResult;
@@ -37,9 +38,9 @@
  */
 final class DefaultSearchService implements SearchIndexServiceInterface
 {
-    private const INDEX_VERSION_ODD = 'odd';
+    public const INDEX_VERSION_ODD = 'odd';
 
-    private const INDEX_VERSION_EVEN = 'even';
+    public const INDEX_VERSION_EVEN = 'even';
 
     use LoggerAwareTrait;
 
@@ -229,7 +230,7 @@ public function createPaginatedSearch(
         int $page,
         int $pageSize,
         bool $aggregationsOnly = false
-    ): AdapterSearchInterface {
+    ): DefaultSearchInterface {
         if ($aggregationsOnly) {
             return new Search(
                 from: 0,

src/SearchIndexAdapter/DefaultSearch/Search/FetchIdsBySearchService.php

@@ -16,11 +16,14 @@
 
 namespace Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search;
 
+use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\FieldCategory;
 use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\FieldCategory\SystemField;
 use Pimcore\Bundle\GenericDataIndexBundle\Exception\InvalidArgumentException;
 use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\DefaultSearchInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\Sort\FieldSort;
 use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\Sort\FieldSortList;
+use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndex\HitData;
+use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndexAdapter\SearchResultHit;
 use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\SearchIndexServiceInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\SearchIndexConfigServiceInterface;
 
@@ -49,6 +52,26 @@ public function fetchAllIds(DefaultSearchInterface $search, string $indexName, b
         return $this->doFetchIds($search, $indexName);
     }
 
+    /**
+     * @return HitData[]
+     */
+    public function fetchAllTypesAndIds(
+        DefaultSearchInterface $search,
+        string $indexName,
+        bool $sortById = true
+    ): array {
+        $search = clone $search;
+        if ($sortById) {
+            $search->setSortList(new FieldSortList([new FieldSort(SystemField::ID->getPath())]));
+        }
+
+        if ($search->getSortList()->isEmpty()) {
+            throw new InvalidArgumentException('Search must have a sort defined to be able to fetch all ids');
+        }
+
+        return $this->doFetchIdsAndTypes($search, $indexName);
+    }
+
     private function doFetchIds(DefaultSearchInterface $search, string $indexName, ?array $searchAfter = null): array
     {
         $search->setFrom(0);
@@ -66,6 +89,34 @@ private function doFetchIds(DefaultSearchInterface $search, string $indexName, ?
         return $ids;
     }
 
+    private function doFetchIdsAndTypes(
+        DefaultSearchInterface $search,
+        string $indexName,
+        ?array $searchAfter = null
+    ): array {
+        $search->setFrom(0);
+        $search->setSize($this->getPageSize());
+        $search->setSource([SystemField::ELEMENT_TYPE->getPath()]);
+        $search->setSearchAfter($searchAfter);
+        $searchResult = $this->searchIndexService->search($search, $indexName);
+        $hits = $searchResult->getHits();
+        $idsAndTypes = array_map(
+            static fn (SearchResultHit $item) =>
+            new HitData(
+                id: $item->getId(),
+                elementType: $item->getSource()[FieldCategory::SYSTEM_FIELDS->value][SystemField::ELEMENT_TYPE->value],
+                index: $item->getIndex(),
+            ),
+            $hits
+        );
+        $lastHit = $searchResult->getLastHit();
+        if ($lastHit && (count($hits) === $this->getPageSize())) {
+            return array_merge($idsAndTypes, $this->doFetchIdsAndTypes($search, $indexName, $lastHit->getSort()));
+        }
+
+        return $idsAndTypes;
+    }
+
     private function getPageSize(): int
     {
         $maxResultWindow = $this->searchIndexConfigService->getIndexSettings()['max_result_window'];

src/SearchIndexAdapter/DefaultSearch/Search/FetchIdsBySearchServiceInterface.php

@@ -17,8 +17,18 @@
 namespace Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search;
 
 use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\DefaultSearchInterface;
+use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndex\HitData;
 
 interface FetchIdsBySearchServiceInterface
 {
     public function fetchAllIds(DefaultSearchInterface $search, string $indexName, bool $sortById = true): array;
+
+    /**
+     * @return HitData[]
+     */
+    public function fetchAllTypesAndIds(
+        DefaultSearchInterface $search,
+        string $indexName,
+        bool $sortById = true
+    ): array;
 }

src/SearchIndexAdapter/SearchIndexServiceInterface.php

@@ -17,6 +17,7 @@
 namespace Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter;
 
 use Exception;
+use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\DefaultSearchInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Interfaces\AdapterSearchInterface;
 use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndexAdapter\SearchResult;
 
@@ -56,7 +57,7 @@ public function createPaginatedSearch(
         int $page,
         int $pageSize,
         bool $aggregationsOnly = false
-    ): AdapterSearchInterface;
+    ): DefaultSearchInterface;
 
     public function search(AdapterSearchInterface $search, string $indexName): SearchResult;
 

src/Service/ElementService.php

@@ -21,8 +21,10 @@
 use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\ElementType;
 use Pimcore\Bundle\GenericDataIndexBundle\Exception\InvalidElementTypeException;
 use Pimcore\Model\Asset;
+use Pimcore\Model\DataObject;
 use Pimcore\Model\DataObject\AbstractObject;
 use Pimcore\Model\Document;
+use Pimcore\Model\Element\ElementInterface;
 
 /**
  * @internal
@@ -47,6 +49,19 @@ public function getElementByType(int $id, string $type): Asset|AbstractObject|Do
         };
     }
 
+    /**
+     * @throws InvalidElementTypeException
+     */
+    public function getElementType(ElementInterface $element): ElementType
+    {
+        return match (true) {
+            $element instanceof Asset => ElementType::ASSET,
+            $element instanceof Document => ElementType::DOCUMENT,
+            $element instanceof DataObject => ElementType::DATA_OBJECT,
+            default => throw new InvalidElementTypeException('Invalid element type: ' . $element->getType())
+        };
+    }
+
     public function classDefinitionExists(string $name): bool
     {
         try {

src/Service/ElementServiceInterface.php

@@ -16,10 +16,12 @@
 
 namespace Pimcore\Bundle\GenericDataIndexBundle\Service;
 
+use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\ElementType;
 use Pimcore\Bundle\GenericDataIndexBundle\Exception\InvalidElementTypeException;
 use Pimcore\Model\Asset;
 use Pimcore\Model\DataObject\AbstractObject;
 use Pimcore\Model\Document;
+use Pimcore\Model\Element\ElementInterface;
 
 /**
  * @internal
@@ -33,5 +35,10 @@ interface ElementServiceInterface
      */
     public function getElementByType(int $id, string $type): Asset|AbstractObject|Document|null;
 
+    /**
+     * @throws InvalidElementTypeException
+     */
+    public function getElementType(ElementInterface $element): ElementType;
+
     public function classDefinitionExists(string $name): bool;
 }

src/Service/Search/SearchService/Asset/AssetSearchService.php

@@ -52,15 +52,15 @@ public function search(
         AssetSearchInterface $assetSearch,
         PermissionTypes $permissionType = PermissionTypes::LIST
     ): AssetSearchResult {
-        $assetSearch = $this->searchHelper->addSearchRestrictions(
+        $search = $this->searchHelper->addSearchRestrictions(
             search: $assetSearch,
             userPermission: UserPermissionTypes::ASSETS->value,
             workspaceType: AssetWorkspace::WORKSPACE_TYPE,
             permissionType: $permissionType
         );
 
         $searchResult = $this->searchHelper->performSearch(
-            search: $assetSearch,
+            search: $search,
             indexName: $this->assetTypeAdapter->getAliasIndexName()
         );
 
@@ -75,12 +75,12 @@ public function search(
                 items: $this->searchHelper->hydrateSearchResultHits(
                     $searchResult,
                     $childrenCounts,
-                    $assetSearch->getUser()
+                    $search->getUser()
                 ),
                 pagination: $this->paginationInfoService->getPaginationInfoFromSearchResult(
                     searchResult: $searchResult,
-                    page: $assetSearch->getPage(),
-                    pageSize: $assetSearch->getPageSize()
+                    page: $search->getPage(),
+                    pageSize: $search->getPageSize()
                 ),
                 aggregations: $searchResult->getAggregations(),
             );