refactor(testing): improve layout and naming of testing packages (#1654)

* chore(tests): create top-level package; rename subpackages

* chore(tests): add top-level ini file

* chore(tests): update packages and paths in pyproject.toml

* chore(tests): update plugin paths used in top-level conftests

* refactor(tests): update imports for base_types, exceptions and vm

* refactor(tests): update imports for specs, fixtures and forks

* refactor(tests): update imports for tools, types and execution

* refactor(tests): update config, client and remaining imports

* refactor(tests): update string path and package references

* refactor(tests): fix remaining imports for mypy

* chore(tests): ignore type for `TransactionGeneric`'s `chain_id` default

* chore(tests): single py.typed at top package level; add stubs to config

* chore(tests): ruff format

* refactor(tests): update pytest ini files to reflect package renames

* refactor(tests): update imports for base_types/exceptions/vm in tests

* refactor(tests): update imports for tools/types/execution in tests

* refactor(tests): update remaining imports in ./tests/

* chore(tests): ruff format tests

* chore(tests): organize formats

* chore(tests): execution_spec_tests -> eest

* chore(tests): Appease ruff

* chore(tests): Fix tests imports

* chore(tests): Remove some warnings

* refactor(packages/tests): Refactor eest.tools and package exports

* refactor(tests): Refactor eest.tools imports

* refactor(packages): Rename `packages/tests` -> `packages/testing`

* fix(gitignore): Fix `fixtures` ignore

* refactor(packages): Rename module `eest` -> `execution_testing`

* refactor(packages): Rename package `eest` -> `execution-testing`

* fix(testing): update paths used by the jinja2 template loader

* fix(testing): fix location of `env.yaml`

* chore(testing): rename simulator prefix from eest to eels

For consistency with the Hive simulator naming, cf ethereum/hive#1362.

* chore(testing): rename package to ethereum-execution-testing

* refactor(testing): ethereum_test_cli -> cli

* fix(testing): remove unimported `vm` from `__all__`

It's not imported in the top-level init.py file;
doing an `import vm from ..` will cause an attribute error.
This keeps things inline with other sub-packages.

* chore(testing): update `eof.py` docstrings to use `execution_testing.*`

Update docstring references from `ethereum_test_*` to `execution_testing.*`
for consistency with the package rename.

* fix(testing): update docstrings to use `execution_testing.*` (specs)

  Update module references from `ethereum_test_*` to `execution_testing.*`
  in test docstrings and imports. Also fixes import in skipped checklist test
  for when it's re-enabled.

* fix(testing): remove `eels_resolutions.json` from package data

* feat(testing): add `Op` as ergonomic alias for `Opcodes`

Export `Op` from `execution_testing` package root to simplify imports.
Users can now write
`from execution_testing import Op`
instead of
`from execution_testing import Opcodes as Op`.

* refactor(testing): Usages of Opcodes

* fix(uv.lock): `ethereum-execution-testing`

* fix(testing): Fix consume commands

---------

Co-authored-by: kclowes <[email protected]>
Co-authored-by: Mario Vega <[email protected]>

Author: 3 people

.gitignore

@@ -75,7 +75,10 @@ tests/execution-spec-generated-tests
 tests/fixtures
 tests/t8n_testdata
 
-fixtures
+fixtures/
 
 logs/
 *.log
+
+# config (output of `uv run make env`)
+env.yaml

packages/tests/pyproject.toml renamed to packages/testing/pyproject.toml

@@ -6,9 +6,9 @@ requires = [
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "ethereum-execution-tests"
+name = "ethereum-execution-testing"
 version = "1.0.0"
-description = "Ethereum execution client test authoring framework"
+description = "Ethereum execution layer client test generation and runner framework"
 readme = "README.md"
 requires-python = ">=3.11"
 license = "MIT"
@@ -89,42 +89,41 @@ docs = [
 ]
 
 [project.scripts]
-fill = "cli.pytest_commands.fill:fill"
-phil = "cli.pytest_commands.fill:phil"
-execute = "cli.pytest_commands.execute:execute"
-attac = "cli.pytest_commands.execute:execute"
-checkfixtures = "cli.check_fixtures:check_fixtures"
-check_eip_versions = "cli.pytest_commands.check_eip_versions:check_eip_versions"
-consume = "cli.pytest_commands.consume:consume"
-protec = "cli.pytest_commands.consume:consume"
-checklist = "cli.pytest_commands.checklist:checklist"
-generate_checklist_stubs = "cli.generate_checklist_stubs:generate_checklist_stubs"
-genindex = "cli.gen_index:generate_fixtures_index_cli"
-gentest = "cli.gentest:generate"
-eofwrap = "cli.eofwrap:eof_wrap"
-pyspelling_soft_fail = "cli.tox_helpers:pyspelling"
-markdownlintcli2_soft_fail = "cli.tox_helpers:markdownlint"
-order_fixtures = "cli.order_fixtures:order_fixtures"
-evm_bytes = "cli.evm_bytes:evm_bytes"
-hasher = "cli.hasher:main"
-eest = "cli.eest.cli:eest"
-fillerconvert = "cli.fillerconvert.fillerconvert:main"
-groupstats = "cli.show_pre_alloc_group_stats:main"
-extract_config = "cli.extract_config:extract_config"
-compare_fixtures = "cli.compare_fixtures:main"
-modify_static_test_gas_limits = "cli.modify_static_test_gas_limits:main"
+fill = "execution_testing.cli.pytest_commands.fill:fill"
+phil = "execution_testing.cli.pytest_commands.fill:phil"
+execute = "execution_testing.cli.pytest_commands.execute:execute"
+attac = "execution_testing.cli.pytest_commands.execute:execute"
+checkfixtures = "execution_testing.cli.check_fixtures:check_fixtures"
+check_eip_versions = "execution_testing.cli.pytest_commands.check_eip_versions:check_eip_versions"
+consume = "execution_testing.cli.pytest_commands.consume:consume"
+protec = "execution_testing.cli.pytest_commands.consume:consume"
+checklist = "execution_testing.cli.pytest_commands.checklist:checklist"
+generate_checklist_stubs = "execution_testing.cli.generate_checklist_stubs:generate_checklist_stubs"
+genindex = "execution_testing.cli.gen_index:generate_fixtures_index_cli"
+gentest = "execution_testing.cli.gentest:generate"
+eofwrap = "execution_testing.cli.eofwrap:eof_wrap"
+pyspelling_soft_fail = "execution_testing.cli.tox_helpers:pyspelling"
+markdownlintcli2_soft_fail = "execution_testing.cli.tox_helpers:markdownlint"
+order_fixtures = "execution_testing.cli.order_fixtures:order_fixtures"
+evm_bytes = "execution_testing.cli.evm_bytes:evm_bytes"
+hasher = "execution_testing.cli.hasher:main"
+eest = "execution_testing.cli.eest.cli:eest"
+fillerconvert = "execution_testing.cli.fillerconvert.fillerconvert:main"
+groupstats = "execution_testing.cli.show_pre_alloc_group_stats:main"
+extract_config = "execution_testing.cli.extract_config:extract_config"
+compare_fixtures = "execution_testing.cli.compare_fixtures:main"
+modify_static_test_gas_limits = "execution_testing.cli.modify_static_test_gas_limits:main"
 
 [tool.setuptools.packages.find]
 where = ["src"]
 exclude = ["*tests*"]
 
 [tool.setuptools.package-data]
-ethereum_test_forks = ["forks/contracts/*.bin"]
-"pytest_plugins.execute" = ["eth_config/networks.yml"]
-"pytest_plugins" = ["eels_resolutions.json"]
-"cli.eest.make" = ["templates/*.j2"]
-"cli.pytest_commands" = ["pytest_ini_files/*.ini"]
-"ethereum_test_types" = ["kzg_trusted_setup.txt"]
+"execution_testing.forks" = ["forks/contracts/*.bin"]
+"execution_testing.cli.pytest_commands.plugins.execute" = ["eth_config/networks.yml"]
+"execution_testing.cli.eest.make" = ["templates/*.j2"]
+"execution_testing.cli.pytest_commands" = ["pytest_ini_files/*.ini"]
+"execution_testing.test_types" = ["kzg_trusted_setup.txt"]
 
 [tool.ruff]
 line-length = 79
@@ -145,9 +144,15 @@ testpaths = [
 ]
 addopts = [
     "-p", "pytester",
-    "--ignore=src/pytest_plugins/consume/test_cache.py",
-    "--ignore=src/pytest_plugins/consume/direct/",
-    "--ignore=src/pytest_plugins/consume/simulators/",
+    "--ignore=src/execution_testing/cli/pytest_commands/plugins/consume/test_cache.py",
+    "--ignore=src/execution_testing/cli/pytest_commands/plugins/consume/direct/",
+    "--ignore=src/execution_testing/cli/pytest_commands/plugins/consume/simulators/",
+]
+markers = [
+    "pre_alloc_group: Custom marker for pre-allocation grouping",
+    "some_mark: Test marker for parametrizer tests",
+    "eip_checklist: Custom marker for EIP checklist tests",
+    "slow: Marks tests as slow running",
 ]
 
 [tool.uv]

packages/tests/src/conftest.py renamed to packages/testing/src/conftest.py

@@ -5,7 +5,7 @@
 
 import pytest
 
-from ethereum_clis import (
+from execution_testing.client_clis import (
     BesuTransitionTool,
     ExecutionSpecsTransitionTool,
     TransitionTool,

packages/tests/src/ethereum_test_tools/__init__.py renamed to packages/testing/src/execution_testing/__init__.py

@@ -1,9 +1,6 @@
-"""
-Module containing tools for generating cross-client Ethereum execution layer
-tests.
-"""
+"""Framework to generate cross-client Ethereum execution layer tests."""
 
-from ethereum_test_base_types import (
+from .base_types import (
     AccessList,
     Account,
     Address,
@@ -15,52 +12,57 @@
     TestPrivateKey,
     TestPrivateKey2,
 )
-from ethereum_test_base_types.reference_spec import (
+from .base_types.reference_spec import (
     ReferenceSpec,
     ReferenceSpecTypes,
 )
-from ethereum_test_benchmark import (
+from .benchmark import (
     BenchmarkCodeGenerator,
     ExtCallGenerator,
     JumpLoopGenerator,
 )
-from ethereum_test_exceptions import (
+from .checklists import EIPChecklist
+from .exceptions import (
     BlockException,
     EngineAPIError,
     EOFException,
     TransactionException,
 )
-from ethereum_test_fixtures import BaseFixture, FixtureCollector
-from ethereum_test_specs import (
+from .fixtures import BaseFixture, FixtureCollector
+from .forks import Fork, GasCosts
+from .specs import (
     BaseTest,
     BenchmarkTest,
     BenchmarkTestFiller,
     BlobsTest,
     BlobsTestFiller,
+    Block,
     BlockchainTest,
     BlockchainTestFiller,
     EOFStateTest,
     EOFStateTestFiller,
     EOFTest,
     EOFTestFiller,
+    Header,
     StateTest,
     StateTestFiller,
     TransactionTest,
     TransactionTestFiller,
 )
-from ethereum_test_specs.blockchain import Block, Header
-from ethereum_test_types import (
+from .test_types import (
     EOA,
     Alloc,
     AuthorizationTuple,
     BalAccountChange,
+    BalAccountExpectation,
     BalBalanceChange,
     BalCodeChange,
     BalNonceChange,
     BalStorageChange,
     BalStorageSlot,
     Blob,
     BlockAccessList,
+    BlockAccessListExpectation,
     ChainConfig,
     ConsolidationRequest,
     DepositRequest,
@@ -69,8 +71,10 @@
     Removable,
     Requests,
     TestParameterGroup,
+    TestPhaseManager,
     Transaction,
     TransactionReceipt,
+    TransactionType,
     Withdrawal,
     WithdrawalRequest,
     add_kzg_version,
@@ -80,42 +84,42 @@
     compute_eofcreate_address,
     keccak256,
 )
-from ethereum_test_vm import (
+from .tools import (
+    CalldataCase,
+    Case,
+    CodeGasMeasure,
+    Conditional,
+    DeploymentTestType,
+    Initcode,
+    ParameterSet,
+    Switch,
+    While,
+    extend_with_defaults,
+    generate_system_contract_deploy_test,
+    generate_system_contract_error_test,
+)
+from .vm import (
     Bytecode,
     EVMCodeType,
     Macro,
     Macros,
     MemoryVariable,
+    Op,
     Opcode,
     OpcodeCallArg,
     Opcodes,
     UndefinedOpcodes,
     call_return_code,
 )
 
-from .tools_code import (
-    CalldataCase,
-    Case,
-    CodeGasMeasure,
-    Conditional,
-    Initcode,
-    Switch,
-    While,
-)
-from .utility.generators import (
-    DeploymentTestType,
-    generate_system_contract_deploy_test,
-    generate_system_contract_error_test,
-)
-from .utility.pytest import extend_with_defaults
-
 __all__ = (
     "AccessList",
     "Account",
     "Address",
     "Alloc",
     "AuthorizationTuple",
     "BalAccountChange",
+    "BalAccountExpectation",
     "BalBalanceChange",
     "BalCodeChange",
     "BalNonceChange",
@@ -127,10 +131,11 @@
     "BenchmarkTest",
     "BenchmarkTestFiller",
     "Blob",
-    "BlockAccessList",
     "BlobsTest",
     "BlobsTestFiller",
     "Block",
+    "BlockAccessList",
+    "BlockAccessListExpectation",
     "BlockchainTest",
     "BlockchainTestFiller",
     "BlockException",
@@ -145,6 +150,7 @@
     "ExtCallGenerator",
     "DeploymentTestType",
     "DepositRequest",
+    "EIPChecklist",
     "EngineAPIError",
     "Environment",
     "EOA",
@@ -155,6 +161,8 @@
     "EOFTestFiller",
     "EVMCodeType",
     "FixtureCollector",
+    "Fork",
+    "GasCosts",
     "Hash",
     "Header",
     "Initcode",
@@ -163,9 +171,11 @@
     "Macros",
     "MemoryVariable",
     "NetworkWrappedTransaction",
+    "Op",
     "Opcode",
     "OpcodeCallArg",
     "Opcodes",
+    "ParameterSet",
     "ReferenceSpec",
     "ReferenceSpecTypes",
     "Removable",
@@ -177,13 +187,15 @@
     "TestAddress",
     "TestAddress2",
     "TestParameterGroup",
+    "TestPhaseManager",
     "TestPrivateKey",
     "TestPrivateKey2",
     "Transaction",
     "TransactionException",
     "TransactionReceipt",
     "TransactionTest",
     "TransactionTestFiller",
+    "TransactionType",
     "UndefinedOpcodes",
     "While",
     "Withdrawal",
@@ -198,5 +210,4 @@
     "generate_system_contract_deploy_test",
     "generate_system_contract_error_test",
     "keccak256",
-    "vm",
 )