COCO mAP metric (#2901)

* Keep only cocomap-related changes

i.e. ObjectDetectionMap and its dependencies

* Some improvements

Removed allow_multiple...
Renamed average_operand
Renamed _measure_recall... to _compute_recall...

* Update docs

* Fix a bug in docs

Docs has some nasty errors

* Fix a tiny bug related to allgather

* Fix a few bugs

* Redesign code:

Removed generic detection logics. Just that of the COCO is remained
Tests are updated

* Remove all_gather with different shape

* Add test for all_gather_with_different_shape func

* A few improvements

* Add an output transform

and apply a review comment

* Add a test for the output_transform

* Remove 'flavor' because all DeciAI, Ultralytics, Detectron and pycocotools use the 'max-precision' approach

* Revert Metric change and a few bug fix

* A tiny improvement in local variable names

* Add max_dep and area_range

* some improvements

* Improvement in code

* Some improvements

* Fix a bug; Some improvements; Improve docs

* Fix metrics.rst

* Remove @OverRide which is for 3.12

* Fix mypy issues

* Fix two tests

* Fix a typo in tests

* Fix dist tests

* Add common obj. det. metrics

* Change an annotation for the sake of M1 python3.8

* Use if check on torch.double usages for MPS backend

* Fix a typo

* Fix a bug related to tensors on same devices

* Fix a bug related to MPS and torch.double

* Fix a bug related to MPS

* Fix a bug related to MPS

* Fix a bug related to MPS

* Resolve MPS's lack of cummax

* Revert MPS fallback

* Apply comments

* Revert unnecessary changes

* Apply review comments

* Skip MPS on test_integraion as well

---------

Co-authored-by: vfdev <[email protected]>

Author: sadra-barikbin

docs/source/metrics.rst

@@ -332,13 +332,17 @@ Complete list of metrics
     Frequency
     Loss
     MeanAbsoluteError
+    MeanAveragePrecision
     MeanPairwiseDistance
     MeanSquaredError
     metric.Metric
     metric_group.MetricGroup
     metrics_lambda.MetricsLambda
     MultiLabelConfusionMatrix
     MutualInformation
+    ObjectDetectionAvgPrecisionRecall
+    CommonObjectDetectionMetrics
+    vision.object_detection_average_precision_recall.coco_tensor_list_to_dict_list
     precision.Precision
     PSNR
     recall.Recall

ignite/metrics/__init__.py

@@ -21,6 +21,7 @@
 from ignite.metrics.loss import Loss
 from ignite.metrics.maximum_mean_discrepancy import MaximumMeanDiscrepancy
 from ignite.metrics.mean_absolute_error import MeanAbsoluteError
+from ignite.metrics.mean_average_precision import MeanAveragePrecision
 from ignite.metrics.mean_pairwise_distance import MeanPairwiseDistance
 from ignite.metrics.mean_squared_error import MeanSquaredError
 from ignite.metrics.metric import BatchFiltered, BatchWise, EpochWise, Metric, MetricUsage
@@ -39,6 +40,11 @@
 from ignite.metrics.running_average import RunningAverage
 from ignite.metrics.ssim import SSIM
 from ignite.metrics.top_k_categorical_accuracy import TopKCategoricalAccuracy
+from ignite.metrics.vision.object_detection_average_precision_recall import (
+    coco_tensor_list_to_dict_list,
+    CommonObjectDetectionMetrics,
+    ObjectDetectionAvgPrecisionRecall,
+)
 
 __all__ = [
     "Metric",
@@ -90,4 +96,8 @@
     "PrecisionRecallCurve",
     "RocCurve",
     "ROC_AUC",
+    "MeanAveragePrecision",
+    "ObjectDetectionAvgPrecisionRecall",
+    "CommonObjectDetectionMetrics",
+    "coco_tensor_list_to_dict_list",
 ]

ignite/metrics/mean_average_precision.py

@@ -873,7 +873,7 @@ def _is_list_of_tensors_or_numbers(x: Sequence[Union[torch.Tensor, float]]) -> b
     return isinstance(x, Sequence) and all([isinstance(t, (torch.Tensor, Number)) for t in x])
 
 
-def _to_batched_tensor(x: Union[torch.Tensor, float], device: Optional[torch.device] = None) -> torch.Tensor:
+def _to_batched_tensor(x: Union[torch.Tensor, Number], device: Optional[torch.device] = None) -> torch.Tensor:
     if isinstance(x, torch.Tensor):
         return x.unsqueeze(dim=0)
     return torch.tensor([x], device=device)

ignite/metrics/metric.py

@@ -1,4 +1,4 @@
-from typing import Any, Callable, Dict, Sequence
+from typing import Any, Callable, Dict, Sequence, Tuple
 
 import torch
 
@@ -15,6 +15,11 @@ class MetricGroup(Metric):
             :class:`~ignite.engine.engine.Engine`'s ``process_function``'s output into the
             form expected by the metric. `output_transform` of each metric in the group is also
             called upon its update.
+        skip_unrolling: specifies whether output should be unrolled before being fed to update method. Should be
+            true for multi-output model, for example, if ``y_pred`` and ``y`` contain multi-ouput as
+            ``(y_pred_a, y_pred_b)`` and ``(y_a, y_b)``, in which case the update method is called for
+            ``(y_pred_a, y_a)`` and ``(y_pred_b, y_b)``.Alternatively, ``output_transform`` can be used to handle
+            this.
 
     Examples:
         We construct a group of metrics, attach them to the engine at once and retrieve their result.
@@ -34,13 +39,18 @@ class MetricGroup(Metric):
 
            # And also altogether
            state.metrics["eval_metrics"]
+
+    .. versionchanged:: 0.5.2
+        ``skip_unrolling`` argument is added.
     """
 
-    _state_dict_all_req_keys = ("metrics",)
+    _state_dict_all_req_keys: Tuple[str, ...] = ("metrics",)
 
-    def __init__(self, metrics: Dict[str, Metric], output_transform: Callable = lambda x: x):
+    def __init__(
+        self, metrics: Dict[str, Metric], output_transform: Callable = lambda x: x, skip_unrolling: bool = False
+    ):
         self.metrics = metrics
-        super(MetricGroup, self).__init__(output_transform=output_transform)
+        super(MetricGroup, self).__init__(output_transform=output_transform, skip_unrolling=skip_unrolling)
 
     def reset(self) -> None:
         for m in self.metrics.values():

ignite/metrics/metric_group.py

@@ -0,0 +1,3 @@
+from ignite.metrics.vision.object_detection_average_precision_recall import ObjectDetectionAvgPrecisionRecall
+
+__all__ = ["ObjectDetectionAvgPrecisionRecall"]