FEATURE: support for different output formats, with differing documentRenderers and custom enumerators

Classes/BackendUi/BackendUiDataService.php

@@ -13,7 +13,7 @@
 use Flowpack\DecoupledContentStore\NodeEnumeration\Domain\Repository\RedisEnumerationRepository;
 use Flowpack\DecoupledContentStore\NodeRendering\Dto\RenderingStatistics;
 use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingErrorManager;
-use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingStatisticsStore;
+use Flowpack\DecoupledContentStore\NodeRendering\Infrastructure\RedisRenderingTimeStatisticsStore;
 use Flowpack\DecoupledContentStore\PrepareContentRelease\Infrastructure\RedisContentReleaseService;
 use Flowpack\DecoupledContentStore\ReleaseSwitch\Infrastructure\RedisReleaseSwitchService;
 use Neos\Flow\Annotations as Flow;
@@ -44,7 +44,7 @@ class BackendUiDataService
 
     /**
      * @Flow\Inject
-     * @var RedisRenderingStatisticsStore
+     * @var RedisRenderingTimeStatisticsStore
      */
     protected $redisRenderingStatisticsStore;
 

Classes/Command/ContentReleaseEventsCommandController.php

@@ -0,0 +1,58 @@
+<?php
+declare(strict_types=1);
+
+namespace Flowpack\DecoupledContentStore\Command;
+
+use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\PrunnerJobId;
+use Flowpack\DecoupledContentStore\Core\Infrastructure\RedisStatisticsEventService;
+use Neos\Flow\Annotations as Flow;
+use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
+use Flowpack\DecoupledContentStore\Core\Infrastructure\ContentReleaseLogger;
+use Neos\Flow\Cli\CommandController;
+
+/**
+ * Commands to read statistics events for a content release from redis
+ */
+class ContentReleaseEventsCommandController extends CommandController
+{
+    #[Flow\Inject]
+    protected RedisStatisticsEventService $redisStatisticsEventService;
+
+    /**
+     * Count statistics events in a content release.
+     *
+     * This command will count how many statics events with given filters, grouped by the specified keys exist in a content release.
+     *
+     * Use --where to filter the events (e.g. "--where=event=title") and --groupBy to count events separately
+     * (e.g. "--groupBy=additionalPayload.preset"). You can specify multiple filters and groups by separating them with
+     * ',' (e.g "--groupBy=additionalPayloads.preset,additionalPayloads.documentId")
+     *
+     * Do not use "--where event=title"! Flow will remove "event=" and the filter will not be applied.
+     *
+     * @param string $contentReleaseIdentifier The contentReleaseIdentifier which statistics should be counted
+     * @param string $where filter the events before counting
+     * @param string $groupBy group the events by this value into separately counted groups
+     * @return void
+     */
+    public function countStatisticsEventCommand(string $contentReleaseIdentifier, string $where = '', string $groupBy = ''): void
+    {
+        $contentReleaseIdentifier = ContentReleaseIdentifier::fromString($contentReleaseIdentifier);
+        // split every string in $where by the first '=' and use the left part as key and the right part as value
+        $where = $where ? array_column(array_map(fn($s) => explode('=', $s, 2), explode(',', $where)), 1, 0) : [];
+        $groupBy = $groupBy ? explode(',', $groupBy) : [];
+
+        $this->output("Filters: \n");
+        if($where) {
+            foreach ($where as $key=>$value) {
+                $this->output("  $key = \"$value\"\n");
+            }
+        } else {
+            $this->output("  None \n");
+        }
+
+        $eventCounts = $this->redisStatisticsEventService->countEvents($contentReleaseIdentifier, $where, $groupBy);
+        $this->output->outputTable($eventCounts, array_merge(['count'], $groupBy));
+
+        $this->output("Total: %d\n", [array_sum(array_map(fn($e) => $e['count'], $eventCounts))]);
+    }
+}

Classes/Core/Infrastructure/ConsoleStatisticsEventOutput.php

@@ -0,0 +1,33 @@
+<?php
+declare(strict_types=1);
+
+namespace Flowpack\DecoupledContentStore\Core\Infrastructure;
+
+use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
+use Neos\Flow\Cli\ConsoleOutput;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class ConsoleStatisticsEventOutput implements StatisticsEventOutputInterface
+{
+    protected OutputInterface $output;
+
+    function __construct(OutputInterface $output)
+    {
+        $this->output = $output;
+    }
+
+    public static function fromConsoleOutput(ConsoleOutput $output): self
+    {
+        return static::fromSymfonyOutput($output->getOutput());
+    }
+
+    public static function fromSymfonyOutput(OutputInterface $output): self
+    {
+        return new static($output);
+    }
+
+    public function writeEvent(ContentReleaseIdentifier $contentReleaseIdentifier, string $prefix, string $event, array $additionalPayload): void
+    {
+        $this->output->writeln($prefix . 'STATISTICS EVENT ' . $event . ($additionalPayload ? ' ' . json_encode($additionalPayload) : ''));
+    }
+}

Classes/Core/Infrastructure/ContentReleaseLogger.php

@@ -14,6 +14,11 @@ class ContentReleaseLogger
      */
     protected $output;
 
+    /**
+     * @var StatisticsEventOutputInterface
+     */
+    protected $statisticsEventOutput;
+
     /**
      * @var ContentReleaseIdentifier
      */
@@ -24,10 +29,11 @@ class ContentReleaseLogger
      */
     protected $logPrefix = '';
 
-    protected function __construct(OutputInterface $output, ContentReleaseIdentifier $contentReleaseIdentifier, ?RendererIdentifier $rendererIdentifier)
+    protected function __construct(OutputInterface $output, ContentReleaseIdentifier $contentReleaseIdentifier, StatisticsEventOutputInterface $statisticsEventOutput, ?RendererIdentifier $rendererIdentifier)
     {
         $this->output = $output;
         $this->contentReleaseIdentifier = $contentReleaseIdentifier;
+        $this->statisticsEventOutput = $statisticsEventOutput;
         $this->rendererIdentifier = $rendererIdentifier;
         $this->logPrefix = '';
 
@@ -37,14 +43,14 @@ protected function __construct(OutputInterface $output, ContentReleaseIdentifier
     }
 
 
-    public static function fromConsoleOutput(ConsoleOutput $output, ContentReleaseIdentifier $contentReleaseIdentifier): self
+    public static function fromConsoleOutput(ConsoleOutput $output, ContentReleaseIdentifier $contentReleaseIdentifier, StatisticsEventOutputInterface $statisticsEventOutput = new RedisStatisticsEventOutput()): self
     {
-        return new static($output->getOutput(), $contentReleaseIdentifier, null);
+        return new static($output->getOutput(), $contentReleaseIdentifier, $statisticsEventOutput, null);
     }
 
-    public static function fromSymfonyOutput(OutputInterface $output, ContentReleaseIdentifier $contentReleaseIdentifier): self
+    public static function fromSymfonyOutput(OutputInterface $output, ContentReleaseIdentifier $contentReleaseIdentifier, StatisticsEventOutputInterface $statisticsEventOutput = new RedisStatisticsEventOutput()): self
     {
-        return new static($output, $contentReleaseIdentifier, null);
+        return new static($output, $contentReleaseIdentifier, $statisticsEventOutput, null);
     }
 
     public function debug($message, array $additionalPayload = [])
@@ -72,8 +78,13 @@ public function logException(\Exception $exception, string $message, array $addi
         $this->output->writeln($this->logPrefix . $message . "\n\n" . $exception->getMessage() . "\n\n" . $exception->getTraceAsString() . "\n\n" . json_encode($additionalPayload));
     }
 
+    public function logStatisticsEvent(string $event, array $additionalPayload = [])
+    {
+        $this->statisticsEventOutput->writeEvent($this->contentReleaseIdentifier, $this->logPrefix, $event, $additionalPayload);
+    }
+
     public function withRenderer(RendererIdentifier $rendererIdentifier): self
     {
-        return new ContentReleaseLogger($this->output, $this->contentReleaseIdentifier, $rendererIdentifier);
+        return new ContentReleaseLogger($this->output, $this->contentReleaseIdentifier, $this->statisticsEventOutput, $rendererIdentifier);
     }
 }

Classes/Core/Infrastructure/RedisStatisticsEventOutput.php

@@ -0,0 +1,18 @@
+<?php
+declare(strict_types=1);
+
+namespace Flowpack\DecoupledContentStore\Core\Infrastructure;
+
+use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
+use Neos\Flow\Annotations as Flow;
+
+class RedisStatisticsEventOutput implements StatisticsEventOutputInterface
+{
+    #[Flow\Inject]
+    protected RedisStatisticsEventService $redisStatisticsEventService;
+
+    public function writeEvent(ContentReleaseIdentifier $contentReleaseIdentifier, string $prefix, string $event, array $additionalPayload): void
+    {
+        $this->redisStatisticsEventService->addEvent($contentReleaseIdentifier, $prefix, $event, $additionalPayload);
+    }
+}

Classes/Core/Infrastructure/RedisStatisticsEventService.php

@@ -0,0 +1,121 @@
+<?php
+declare(strict_types=1);
+
+namespace Flowpack\DecoupledContentStore\Core\Infrastructure;
+
+use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
+use Flowpack\DecoupledContentStore\Core\RedisKeyService;
+use Flowpack\DecoupledContentStore\Exception;
+use Neos\Flow\Annotations as Flow;
+
+#[Flow\Scope("singleton")]
+class RedisStatisticsEventService
+{
+    #[Flow\Inject]
+    protected RedisClientManager $redisClientManager;
+
+    #[Flow\Inject]
+    protected RedisKeyService $redisKeyService;
+
+    public function addEvent(ContentReleaseIdentifier $contentReleaseIdentifier, string $prefix, string $event, array $additionalPayload): void
+    {
+        $this->redisClientManager->getPrimaryRedis()->rPush($this->redisKeyService->getRedisKeyForPostfix($contentReleaseIdentifier, 'statisticsEvents'), json_encode([
+            'event' => $event,
+            'prefix' => $prefix,
+            'additionalPayload' => $additionalPayload,
+        ]));
+    }
+
+    /**
+     * @param ContentReleaseIdentifier $contentReleaseIdentifier
+     * @param array<string,string> $where
+     * @param string[] $groupBy
+     * @return array<>
+     * @throws Exception
+     */
+    public function countEvents(
+        ContentReleaseIdentifier $contentReleaseIdentifier,
+        array                    $where,
+        array                    $groupBy,
+    ): array
+    {
+        $redis = $this->redisClientManager->getPrimaryRedis();
+        $key = $this->redisKeyService->getRedisKeyForPostfix($contentReleaseIdentifier, 'statisticsEvents');
+        $chunkSize = 1000;
+
+        $countedEvents = [];
+
+        $listLength = $redis->lLen($key);
+        for ($start = 0; $start < $listLength; $start += $chunkSize) {
+            $events = $redis->lRange($key, $start, $start + $chunkSize - 1);
+
+            foreach ($events as $eventJson) {
+                $event = $this->flatten(json_decode($eventJson, true));
+                if($this->shouldCount($event, $where)) {
+                    $group = $this->groupValues($event, $groupBy);
+                    $eventKey = json_encode($group);
+                    if (array_key_exists($eventKey, $countedEvents)) {
+                        $countedEvents[$eventKey]['count'] += 1;
+                    } else {
+                        $countedEvents[$eventKey] = array_merge(['count' => 1], $group);
+                    }
+                }
+            }
+        }
+        // throw away the keys and sort in _reverse_ order by count
+        usort($countedEvents, fn($a, $b) => $b['count'] - $a['count']);
+        return $countedEvents;
+    }
+
+    /**
+     * @phpstan-type JSONArray array<string, string|JSONArray>
+     *´
+     * @param JSONArray $array
+     * @return array<string,string>
+     */
+    private function flatten(array $array): array
+    {
+        $results = [];
+
+        foreach ($array as $key => $value) {
+            if (is_array($value) && ! empty($value)) {
+                foreach ($this->flatten($value) as $subKey => $subValue) {
+                    $results[$key . '.' . $subKey] = $subValue;
+                }
+            } else {
+                $results[$key] = $value;
+            }
+        }
+
+        return $results;
+    }
+
+    /**
+     * @param array<string,string> $event
+     * @param array<string,string> $where
+     * @return bool
+     */
+    private function shouldCount(array $event, array $where): bool
+    {
+        foreach ($where as $key=>$value) {
+            if (!array_key_exists($key, $event) || $event[$key] !== $value) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * @param array<string,string> $event
+     * @param string[] $groupedBy
+     * @return array<string,string>
+     */
+    private function groupValues(array $event, array $groupedBy): array
+    {
+        $group = [];
+        foreach ($groupedBy as $path) {
+            $group[$path] = $event[$path] ?? null;
+        }
+        return $group;
+    }
+}

Classes/Core/Infrastructure/StatisticsEventOutputInterface.php

@@ -0,0 +1,11 @@
+<?php
+declare(strict_types=1);
+
+namespace Flowpack\DecoupledContentStore\Core\Infrastructure;
+
+use Flowpack\DecoupledContentStore\Core\Domain\ValueObject\ContentReleaseIdentifier;
+
+interface StatisticsEventOutputInterface
+{
+    public function writeEvent(ContentReleaseIdentifier $contentReleaseIdentifier, string $prefix, string $event, array $additionalPayload): void;
+}

Classes/NodeEnumeration/Domain/Dto/EnumeratedNode.php

@@ -41,17 +41,24 @@ final class EnumeratedNode implements \JsonSerializable
      */
     protected $nodeTypeName;
 
-    private function __construct(string $contextPath, string $nodeIdentifier, string $nodeTypeName, array $arguments)
+    /**
+     * The renderer implementation to use for this EnumeratedNode;
+     * a key from Settings at Flowpack.DecoupledContentStore.extensions.documentRenderers.[key]
+     */
+    public readonly string $rendererId;
+
+    private function __construct(string $contextPath, string $nodeIdentifier, string $nodeTypeName, array $arguments, string $rendererId = '')
     {
         $this->contextPath = $contextPath;
         $this->nodeIdentifier = $nodeIdentifier;
         $this->nodeTypeName = $nodeTypeName;
         $this->arguments = $arguments;
+        $this->rendererId = $rendererId;
     }
 
     static public function fromNode(NodeInterface $node, array $arguments = []): self
     {
-        return new self($node->getContextPath(), $node->getIdentifier(), $node->getNodeType()->getName(), $arguments);
+        return new self($node->getContextPath(), $node->getIdentifier(), $node->getNodeType()->getName(), $arguments, '');
     }
 
     static public function fromJsonString(string $enumeratedNodeString): self
@@ -60,7 +67,7 @@ static public function fromJsonString(string $enumeratedNodeString): self
         if (!is_array($tmp)) {
             throw new \Exception('EnumeratedNode cannot be constructed from: ' . $enumeratedNodeString);
         }
-        return new self($tmp['contextPath'], $tmp['nodeIdentifier'], $tmp['nodeTypeName'] ?? '', $tmp['arguments']);
+        return new self($tmp['contextPath'], $tmp['nodeIdentifier'], $tmp['nodeTypeName'] ?? '', $tmp['arguments'], $tmp['rendererId']);
     }
 
     public function jsonSerialize()
@@ -69,7 +76,8 @@ public function jsonSerialize()
             'contextPath' => $this->contextPath,
             'nodeIdentifier' => $this->nodeIdentifier,
             'nodeTypeName' => $this->nodeTypeName,
-            'arguments' => $this->arguments
+            'arguments' => $this->arguments,
+            'rendererId' => $this->rendererId,
         ];
     }
 
@@ -105,6 +113,11 @@ public function getArguments(): array
 
     public function debugString(): string
     {
-        return sprintf('%s %s %s(%s)', $this->nodeTypeName, $this->nodeIdentifier, $this->arguments ? http_build_query($this->arguments) . ' ' : '', $this->contextPath);
+        return sprintf('%s %s %s(%s) - %s', $this->nodeTypeName, $this->nodeIdentifier, $this->arguments ? http_build_query($this->arguments) . ' ' : '', $this->contextPath, $this->rendererId);
+    }
+
+    public function withRendererId(string $rendererId): self
+    {
+        return new self($this->contextPath, $this->nodeIdentifier, $this->nodeTypeName, $this->arguments, $rendererId);
     }
 }