fix(tests): revamp cache

fix(tests): Don't cache fixtures

Try to implement cache

Fix caching

feat(tests): Manage cache during execution

Author: marioevz

tests/json_infra/conftest.py

@@ -25,7 +25,7 @@
 from requests_cache.backends.sqlite import SQLiteCache
 
 from . import TEST_FIXTURES
-from .helpers import FixturesFile
+from .helpers import FixturesFile, FixtureTestItem
 
 try:
     from xdist import get_xdist_worker_id
@@ -301,3 +301,16 @@ def pytest_collect_file(
     if file_path.suffix == ".json":
         return FixturesFile.from_parent(parent, path=file_path)
     return None
+
+
+def pytest_runtest_teardown(item: Item, nextitem: Item) -> None:
+    """
+    Drop cache from a `FixtureTestItem` if the next one is not of the
+    same type or does not belong to the same fixtures file.
+    """
+    if isinstance(item, FixtureTestItem):
+        if not isinstance(nextitem, FixtureTestItem):
+            item.fixtures_file.clear_data_cache()
+        else:
+            if item.fixtures_file != nextitem.fixtures_file:
+                item.fixtures_file.clear_data_cache()

tests/json_infra/helpers/__init__.py

@@ -1,10 +1,10 @@
 """Helpers to load tests from JSON files."""
 
-from .fixtures import ALL_FIXTURE_TYPES, Fixture, FixturesFile
+from .fixtures import ALL_FIXTURE_TYPES, Fixture, FixturesFile, FixtureTestItem
 from .load_blockchain_tests import BlockchainTestFixture
 from .load_state_tests import StateTestFixture
 
 ALL_FIXTURE_TYPES.append(BlockchainTestFixture)
 ALL_FIXTURE_TYPES.append(StateTestFixture)
 
-__all__ = ["ALL_FIXTURE_TYPES", "Fixture", "FixturesFile"]
+__all__ = ["ALL_FIXTURE_TYPES", "Fixture", "FixturesFile", "FixtureTestItem"]

tests/json_infra/helpers/exceptional_test_patterns.py

@@ -5,7 +5,6 @@
 
 import re
 from dataclasses import dataclass
-from functools import lru_cache
 from typing import Pattern, Tuple
 
 
@@ -21,7 +20,6 @@ class TestPatterns:
     big_memory: Tuple[Pattern[str], ...]
 
 
-@lru_cache
 def exceptional_blockchain_test_patterns(
     json_fork: str, eels_fork: str
 ) -> TestPatterns:
@@ -106,7 +104,6 @@ def exceptional_blockchain_test_patterns(
     )
 
 
-@lru_cache
 def exceptional_state_test_patterns(
     json_fork: str, eels_fork: str
 ) -> TestPatterns:

tests/json_infra/helpers/fixtures.py

@@ -2,12 +2,24 @@
 
 import json
 from abc import ABC, abstractmethod
+from functools import cached_property
 from typing import Any, Dict, Generator, List, Self, Type
 
 from _pytest.nodes import Node
 from pytest import Collector, File, Item
 
 
+class FixtureTestItem(Item):
+    """
+    Test item that comes from a fixture file.
+    """
+
+    @property
+    def fixtures_file(self) -> "FixturesFile":
+        """Return the fixtures file from which the test was extracted."""
+        raise NotImplementedError()
+
+
 class Fixture(ABC):
     """
     Single fixture from a JSON file.
@@ -18,20 +30,17 @@ class Fixture(ABC):
 
     test_file: str
     test_key: str
-    test_dict: Dict[str, Any]
 
     def __init__(
         self,
         *args: Any,
         test_file: str,
         test_key: str,
-        test_dict: Dict[str, Any],
         **kwargs: Any,
     ):
         super().__init__(*args, **kwargs)
         self.test_file = test_file
         self.test_key = test_key
-        self.test_dict = test_dict
 
     @classmethod
     def from_parent(
@@ -57,30 +66,41 @@ def is_format(cls, test_dict: Dict[str, Any]) -> bool:
 class FixturesFile(File):
     """Single JSON file containing fixtures."""
 
+    @cached_property
+    def data(self) -> Dict[str, Any]:
+        """Return the JSON data of the full file."""
+        # loaded once per worker per file (thanks to cached_property)
+        with self.fspath.open("r", encoding="utf-8") as f:
+            return json.load(f)
+
+    def clear_data_cache(self) -> None:
+        """Drop the data cache."""
+        del self.data
+
     def collect(
         self: Self,
     ) -> Generator[Item | Collector, None, None]:
         """Collect test cases from a single JSON fixtures file."""
-        with open(self.path, "r") as file:
-            try:
-                loaded_file = json.load(file)
-            except Exception:
-                return  # Skip *.json files that are unreadable.
-            if not isinstance(loaded_file, dict):
-                return
-            for key, test_dict in loaded_file.items():
-                if not isinstance(test_dict, dict):
+        try:
+            loaded_file = self.data
+        except Exception:
+            return  # Skip *.json files that are unreadable.
+        if not isinstance(loaded_file, dict):
+            return
+        for key, test_dict in loaded_file.items():
+            if not isinstance(test_dict, dict):
+                continue
+            for fixture_type in ALL_FIXTURE_TYPES:
+                if not fixture_type.is_format(test_dict):
                     continue
-                for fixture_type in ALL_FIXTURE_TYPES:
-                    if not fixture_type.is_format(test_dict):
-                        continue
-                    name = key
-                    if "::" in name:
-                        name = name.split("::")[1]
-                    yield fixture_type.from_parent(  # type: ignore
-                        parent=self,
-                        name=name,
-                        test_file=str(self.path),
-                        test_key=key,
-                        test_dict=test_dict,
-                    )
+                name = key
+                if "::" in name:
+                    name = name.split("::")[1]
+                yield fixture_type.from_parent(  # type: ignore
+                    parent=self,
+                    name=name,
+                    test_file=str(self.path),
+                    test_key=key,
+                )
+        # Make sure we don't keep anything from collection in memory.
+        self.clear_data_cache()

tests/json_infra/helpers/load_blockchain_tests.py

@@ -9,7 +9,6 @@
 from ethereum_rlp import rlp
 from ethereum_rlp.exceptions import RLPException
 from ethereum_types.numeric import U64
-from pytest import Item
 
 from ethereum.crypto.hash import keccak256
 from ethereum.exceptions import EthereumException, StateWithEmptyAccount
@@ -18,7 +17,7 @@
 
 from .. import FORKS
 from .exceptional_test_patterns import exceptional_blockchain_test_patterns
-from .fixtures import Fixture
+from .fixtures import Fixture, FixturesFile, FixtureTestItem
 
 
 class NoTestsFoundError(Exception):
@@ -59,7 +58,7 @@ def add_block_to_chain(
             )
 
 
-class BlockchainTestFixture(Fixture, Item):
+class BlockchainTestFixture(Fixture, FixtureTestItem):
     """Single blockchain test fixture from a JSON file."""
 
     fork_name: str
@@ -91,6 +90,20 @@ def __init__(
         if any(x.search(_identifier) for x in test_patterns.big_memory):
             self.add_marker("bigmem")
 
+    @property
+    def fixtures_file(self) -> FixturesFile:
+        """Fixtures file from which the test fixture was collected."""
+        parent = self.parent
+        assert parent is not None
+        assert isinstance(parent, FixturesFile)
+        return parent
+
+    @property
+    def test_dict(self) -> Dict[str, Any]:
+        """Load test from disk."""
+        loaded_file = self.fixtures_file.data
+        return loaded_file[self.test_key]
+
     def runtest(self) -> None:
         """Run a blockchain state test from JSON test case data."""
         json_data = self.test_dict

tests/json_infra/helpers/load_state_tests.py

@@ -12,51 +12,73 @@
 from ethereum.exceptions import StateWithEmptyAccount
 from ethereum.utils.hexadecimal import hex_to_bytes
 from ethereum_spec_tools.evm_tools import create_parser
-from ethereum_spec_tools.evm_tools.statetest import TestCase, read_test_case
+from ethereum_spec_tools.evm_tools.statetest import read_test_case
 from ethereum_spec_tools.evm_tools.t8n import T8N
 
 from .. import FORKS
 from .exceptional_test_patterns import (
     exceptional_state_test_patterns,
 )
-from .fixtures import Fixture
+from .fixtures import Fixture, FixturesFile, FixtureTestItem
 
 parser = create_parser()
 
 
-class StateTest(Item):
+class StateTest(FixtureTestItem):
     """Single state test case item."""
 
-    test_case: TestCase
-    test_dict: Dict[str, Any]
+    index: int
+    fork_name: str
 
     def __init__(
         self,
         *args: Any,
-        test_case: TestCase,
-        test_dict: Dict[str, Any],
+        index: int,
+        fork_name: str,
+        key: str,
         **kwargs: Any,
     ) -> None:
         """Initialize a single test case item."""
         super().__init__(*args, **kwargs)
-        self.test_case = test_case
-        self.test_dict = test_dict
-        self.add_marker(pytest.mark.fork(self.test_case.fork_name))
+        self.index = index
+        self.fork_name = fork_name
+        self.add_marker(pytest.mark.fork(self.fork_name))
         self.add_marker("evm_tools")
         self.add_marker("json_state_tests")
-        eels_fork = FORKS[test_case.fork_name]["eels_fork"]
-        test_patterns = exceptional_state_test_patterns(
-            test_case.fork_name, eels_fork
-        )
-        if any(x.search(test_case.key) for x in test_patterns.slow):
+        eels_fork = FORKS[fork_name]["eels_fork"]
+        test_patterns = exceptional_state_test_patterns(fork_name, eels_fork)
+        if any(x.search(key) for x in test_patterns.slow):
             self.add_marker("slow")
 
+    @property
+    def state_test_fixture(self) -> "StateTestFixture":
+        """Return the state test fixture this test belongs to."""
+        parent = self.parent
+        assert parent is not None
+        assert isinstance(parent, StateTestFixture)
+        return parent
+
+    @property
+    def test_key(self) -> str:
+        """Return the key of the state test fixture in the fixture file."""
+        return self.state_test_fixture.test_key
+
+    @property
+    def fixtures_file(self) -> FixturesFile:
+        """Fixtures file from which the test fixture was collected."""
+        return self.state_test_fixture.fixtures_file
+
+    @property
+    def test_dict(self) -> Dict[str, Any]:
+        """Load test from disk."""
+        loaded_file = self.fixtures_file.data
+        return loaded_file[self.test_key]
+
     def runtest(self) -> None:
         """
         Runs a single general state test.
         """
-        index = self.test_case.index
-        json_fork = self.test_case.fork_name
+        json_fork = self.fork_name
         test_dict = self.test_dict
 
         env = test_dict["env"]
@@ -68,7 +90,7 @@ def runtest(self) -> None:
 
         alloc = test_dict["pre"]
 
-        post = test_dict["post"][json_fork][index]
+        post = test_dict["post"][self.fork_name][self.index]
         post_hash = post["hash"]
         d = post["indexes"]["data"]
         g = post["indexes"]["gas"]
@@ -144,6 +166,20 @@ def is_format(cls, test_dict: Dict[str, Any]) -> bool:
             return False
         return True
 
+    @property
+    def fixtures_file(self) -> FixturesFile:
+        """Fixtures file from which the test fixture was collected."""
+        parent = self.parent
+        assert parent is not None
+        assert isinstance(parent, FixturesFile)
+        return parent
+
+    @property
+    def test_dict(self) -> Dict[str, Any]:
+        """Load test from disk."""
+        loaded_file = self.fixtures_file.data
+        return loaded_file[self.test_key]
+
     def collect(self) -> Iterable[Item | Collector]:
         """Collect state test cases inside of this fixture."""
         for test_case in read_test_case(
@@ -157,6 +193,7 @@ def collect(self) -> Iterable[Item | Collector]:
             yield StateTest.from_parent(
                 parent=self,
                 name=name,
-                test_case=test_case,
-                test_dict=self.test_dict,
+                index=test_case.index,
+                fork_name=test_case.fork_name,
+                key=self.test_key,
             )