Merge #242

242: Make import command resumabe r=brunoocasali a=mkilmanas

# Pull Request
Make it possible to resume import from a mid-point after unsuccessful previous import

## Related issue
Fixes #241

## What does this PR do?
- Makes import resumable:
  - Introduce `--skip-batches` parameter
  - Enforce entity sorting by primary key (consistent, fast)
  - Display the number of processed entities
  - Add test case
- Other small improvements:
  - Clarify wording of the message telling the number of indexed records
- Update existing tests for modified messages

## PR checklist
Please check if your PR fulfills the following requirements:
- [x] Does this PR fix an existing issue, or have you listed the changes applied in the PR description (and why they are needed)?
- [x] Have you read the contributing guidelines?
- [x] Have you made sure that the title is accurate and descriptive of the changes?


Co-authored-by: Marijus Kilmanas <[email protected]>

Author: bors[bot]

src/Command/MeilisearchImportCommand.php

@@ -55,6 +55,13 @@ protected function configure(): void
                 'Update settings related to indices to the search engine'
             )
             ->addOption('batch-size', null, InputOption::VALUE_REQUIRED)
+            ->addOption(
+                'skip-batches',
+                null,
+                InputOption::VALUE_REQUIRED,
+                'Skip the first N batches and start importing from the N+1 batch',
+                0
+            )
             ->addOption(
                 'response-timeout',
                 't',
@@ -86,7 +93,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         }
 
         $entitiesToIndex = array_unique($indexes->all(), SORT_REGULAR);
-        $batchSize = $input->getOption('batch-size');
+        $batchSize = $input->getOption('batch-size') ?? '';
         $batchSize = ctype_digit($batchSize) ? (int) $batchSize : $config->get('batchSize');
         $responseTimeout = ((int) $input->getOption('response-timeout')) ?: self::DEFAULT_RESPONSE_TIMEOUT;
 
@@ -97,29 +104,47 @@ protected function execute(InputInterface $input, OutputInterface $output): int
                 continue;
             }
 
+            $totalIndexed = 0;
+
             $manager = $this->managerRegistry->getManagerForClass($entityClassName);
             $repository = $manager->getRepository($entityClassName);
+            $classMetadata = $manager->getClassMetadata($entityClassName);
+            $entityIdentifiers = $classMetadata->getIdentifierFieldNames();
+            $sortByIdentifiersParam = array_combine($entityIdentifiers, array_fill(0, count($entityIdentifiers), 'ASC'));
 
             $output->writeln('<info>Importing for index '.$entityClassName.'</info>');
 
-            $page = 0;
+            $page = max(0, (int) $input->getOption('skip-batches'));
+
+            if ($page > 0) {
+                $output->writeln(
+                    sprintf(
+                        '<info>Skipping first <comment>%d</comment> batches (<comment>%d</comment> records)</info>',
+                        $page,
+                        $page * $batchSize,
+                    )
+                );
+            }
+
             do {
                 $entities = $repository->findBy(
                     [],
-                    null,
+                    $sortByIdentifiersParam,
                     $batchSize,
                     $batchSize * $page
                 );
 
                 $responses = $this->formatIndexingResponse($this->searchService->index($manager, $entities), $responseTimeout);
+                $totalIndexed += count($entities);
                 foreach ($responses as $indexName => $numberOfRecords) {
                     $output->writeln(
                         sprintf(
-                            'Indexed <comment>%s / %s</comment> %s entities into %s index',
+                            'Indexed a batch of <comment>%d / %d</comment> %s entities into %s index (%d indexed since start)',
                             $numberOfRecords,
                             count($entities),
                             $entityClassName,
-                            '<info>'.$indexName.'</info>'
+                            '<info>'.$indexName.'</info>',
+                            $totalIndexed,
                         )
                     );
                 }

tests/Integration/CommandsTest.php

@@ -74,25 +74,25 @@ public function testSearchImportAndClearAndDeleteWithoutIndices(): void
 
         $this->assertSame(<<<'EOD'
 Importing for index Meilisearch\Bundle\Tests\Entity\Post
-Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__posts index
-Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__aggregated index
+Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__posts index (6 indexed since start)
+Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__aggregated index (6 indexed since start)
 Settings updated.
 Settings updated.
 Settings updated.
 Importing for index Meilisearch\Bundle\Tests\Entity\Comment
 Importing for index Meilisearch\Bundle\Tests\Entity\Tag
-Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__tags index
-Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__aggregated index
+Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__tags index (6 indexed since start)
+Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__aggregated index (6 indexed since start)
 Importing for index Meilisearch\Bundle\Tests\Entity\Link
 Importing for index Meilisearch\Bundle\Tests\Entity\Page
-Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index
+Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (6 indexed since start)
 Importing for index Meilisearch\Bundle\Tests\Entity\SelfNormalizable
 Importing for index Meilisearch\Bundle\Tests\Entity\Post
-Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__posts index
-Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__aggregated index
+Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__posts index (6 indexed since start)
+Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__aggregated index (6 indexed since start)
 Importing for index Meilisearch\Bundle\Tests\Entity\Tag
-Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__tags index
-Indexed 6 / 6 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__aggregated index
+Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__tags index (6 indexed since start)
+Indexed a batch of 6 / 6 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__aggregated index (6 indexed since start)
 Done!
 
 EOD, $importOutput);
@@ -150,12 +150,12 @@ public function testSearchImportWithCustomBatchSize(): void
 
         $this->assertSame(<<<'EOD'
 Importing for index Meilisearch\Bundle\Tests\Entity\Page
-Indexed 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index
-Indexed 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index
-Indexed 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index
-Indexed 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index
-Indexed 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index
-Indexed 1 / 1 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index
+Indexed a batch of 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (2 indexed since start)
+Indexed a batch of 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (4 indexed since start)
+Indexed a batch of 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (6 indexed since start)
+Indexed a batch of 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (8 indexed since start)
+Indexed a batch of 2 / 2 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (10 indexed since start)
+Indexed a batch of 1 / 1 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (11 indexed since start)
 Done!
 
 EOD, $importOutput);
@@ -176,7 +176,7 @@ public function testSearchImportWithCustomResponseTimeout(): void
         $output = $importCommandTester->getDisplay();
 
         $this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Page', $output);
-        $this->assertStringContainsString('Indexed '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index', $output);
+        $this->assertStringContainsString('Indexed a batch of '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index ('.$i.' indexed since start)', $output);
         $this->assertStringContainsString('Done!', $output);
         $this->assertSame(0, $return);
 
@@ -197,7 +197,7 @@ public function testSearchImportWithCustomResponseTimeout(): void
         $output = $importCommandTester->getDisplay();
 
         $this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Page', $output);
-        $this->assertStringContainsString('Indexed '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index', $output);
+        $this->assertStringContainsString('Indexed a batch of '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index ('.$i.' indexed since start)', $output);
         $this->assertStringContainsString('Done!', $output);
         $this->assertSame(0, $return);
     }
@@ -221,8 +221,8 @@ public function testImportDifferentEntitiesIntoSameIndex(): void
 
         $output = $commandTester->getDisplay();
         $this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Tag', $output);
-        $this->assertStringContainsString('Indexed '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__tags index', $output);
-        $this->assertStringContainsString('Indexed 2 / 2 Meilisearch\Bundle\Tests\Entity\Link entities into sf_phpunit__tags index', $output);
+        $this->assertStringContainsString('Indexed a batch of '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__tags index ('.$i.' indexed since start)', $output);
+        $this->assertStringContainsString('Indexed a batch of 2 / 2 Meilisearch\Bundle\Tests\Entity\Link entities into sf_phpunit__tags index (2 indexed since start)', $output);
         $this->assertStringContainsString('Done!', $output);
 
         /** @var SearchResult $searchResult */
@@ -245,7 +245,30 @@ public function testSearchImportAggregator(): void
 
         $output = $commandTester->getDisplay();
         $this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Post', $output);
-        $this->assertStringContainsString('Indexed '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__'.self::$indexName.' index', $output);
+        $this->assertStringContainsString('Indexed a batch of '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__'.self::$indexName.' index ('.$i.' indexed since start)', $output);
+        $this->assertStringContainsString('Done!', $output);
+        $this->assertSame(0, $return);
+    }
+
+    public function testSearchImportWithSkipBatches(): void
+    {
+        for ($i = 0; $i < 10; ++$i) {
+            $this->createPage($i);
+        }
+
+        $command = $this->application->find('meili:import');
+        $commandTester = new CommandTester($command);
+        $return = $commandTester->execute([
+            '--indices' => 'pages',
+            '--batch-size' => '3',
+            '--skip-batches' => '2',
+        ]);
+
+        $output = $commandTester->getDisplay();
+        $this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Page', $output);
+        $this->assertStringContainsString('Skipping first 2 batches (6 records)', $output);
+        $this->assertStringContainsString('Indexed a batch of 3 / 3 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (3 indexed since start)', $output);
+        $this->assertStringContainsString('Indexed a batch of 1 / 1 Meilisearch\Bundle\Tests\Entity\Page entities into sf_phpunit__pages index (4 indexed since start)', $output);
         $this->assertStringContainsString('Done!', $output);
         $this->assertSame(0, $return);
     }
@@ -264,7 +287,7 @@ public function testImportingIndexNameWithAndWithoutPrefix(): void
 
         $output = $commandTester->getDisplay();
         $this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Post', $output);
-        $this->assertStringContainsString('Indexed '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__'.self::$indexName.' index', $output);
+        $this->assertStringContainsString('Indexed a batch of '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__'.self::$indexName.' index ('.$i.' indexed since start)', $output);
         $this->assertStringContainsString('Done!', $output);
         $this->assertSame(0, $return);
 
@@ -283,7 +306,7 @@ public function testImportingIndexNameWithAndWithoutPrefix(): void
 
         $output = $commandTester->getDisplay();
         $this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Post', $output);
-        $this->assertStringContainsString('Indexed '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__'.self::$indexName.' index', $output);
+        $this->assertStringContainsString('Indexed a batch of '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__'.self::$indexName.' index ('.$i.' indexed since start)', $output);
         $this->assertStringContainsString('Done!', $output);
         $this->assertSame(0, $return);
     }
@@ -352,7 +375,7 @@ public function testImportsSelfNormalizable(): void
 
         $this->assertSame(<<<'EOD'
 Importing for index Meilisearch\Bundle\Tests\Entity\SelfNormalizable
-Indexed 2 / 2 Meilisearch\Bundle\Tests\Entity\SelfNormalizable entities into sf_phpunit__self_normalizable index
+Indexed a batch of 2 / 2 Meilisearch\Bundle\Tests\Entity\SelfNormalizable entities into sf_phpunit__self_normalizable index (2 indexed since start)
 Done!
 
 EOD, $importOutput);

tests/Integration/SearchTest.php

@@ -69,10 +69,10 @@ public function testSearchImportAggregator(): void
 
         $this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Post', $output);
         $this->assertStringContainsString('Importing for index Meilisearch\Bundle\Tests\Entity\Tag', $output);
-        $this->assertStringContainsString('Indexed '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__posts index', $output);
-        $this->assertStringContainsString('Indexed '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__'.self::$indexName.' index', $output);
-        $this->assertStringContainsString('Indexed 1 / 1 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__tags index', $output);
-        $this->assertStringContainsString('Indexed 1 / 1 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__'.self::$indexName.' index', $output);
+        $this->assertStringContainsString('Indexed a batch of '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__posts index ('.$i.' indexed since start)', $output);
+        $this->assertStringContainsString('Indexed a batch of '.$i.' / '.$i.' Meilisearch\Bundle\Tests\Entity\Post entities into sf_phpunit__'.self::$indexName.' index ('.$i.' indexed since start)', $output);
+        $this->assertStringContainsString('Indexed a batch of 1 / 1 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__tags index (1 indexed since start)', $output);
+        $this->assertStringContainsString('Indexed a batch of 1 / 1 Meilisearch\Bundle\Tests\Entity\Tag entities into sf_phpunit__'.self::$indexName.' index (1 indexed since start)', $output);
         $this->assertStringContainsString('Done!', $output);
 
         $searchTerm = 'Test';