diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d67a1756..43c3bdf7 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -20,6 +20,19 @@ "ghcr.io/devcontainers-contrib/features/pre-commit:2": {}, "ghcr.io/devcontainers/features/github-cli:1": {} }, + "remoteEnv": { + // ------------------------------------------------- + // ENV settings to preserve history between rebuilds + "PROMPT_COMMAND": "history -a", + "HISTFILE": "/.devcontainercache/.bash_history", + // ------------------------------------------------- + // CACHE Folders + "PRE_COMMIT_HOME": "/.devcontainercache/pre-commit", + "PIP_CACHE_DIR": "/.devcontainercache/pip-cache" + }, + "mounts": [ + "source=${devcontainerId},target=/.devcontainercache,type=volume" + ], // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], @@ -34,7 +47,8 @@ "GitHub.copilot", "ms-python.python", "ms-dotnettools.csharp", - "ryanluker.vscode-coverage-gutters" + "ryanluker.vscode-coverage-gutters", + "charliermarsh.ruff" ] } } diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index 86d659b6..995120c4 100755 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +sudo chown -R $(id -u):$(id -g) /.devcontainercache + dotnet build poetry install --no-interaction --no-root poetry run pip install -e . diff --git a/docs/advanced/overriding_expression_functions.md b/docs/advanced/overriding_expression_functions.md index 7f17725b..a0e74a60 100644 --- a/docs/advanced/overriding_expression_functions.md +++ b/docs/advanced/overriding_expression_functions.md @@ -1,9 +1,90 @@ # Overriding expression functions -The framework interprets expressions containing functions, which are implemented within the framework, and they might contain bugs. -You can override their implementation as illustrated below: +The framework supports mocking non-deterministic expression functions, such as as `rand` or `utcnow` by +providing mock results for them. This can be useful for non-deterministic functions or for testing purposes. + +## Providing mock results for Expression Functions + +You can provide mock results in two ways: + +1. **Using the `data_factory_testing_framework.mock_helpers` module**: This provides the recommended and most convenient way to mock functions by passing either a single value or a list of values that will be returned in sequence. +2. **Using the `Mock` class directly**: This allows to create a mock object directly and set its return value. + +When providing mock results, you can specify a **single value** or a **list of values**. If you provide a list, the mock will return the values in sequence for each call to the function similar to how `unittest.mock` works. + +### Single Value Mocking + +Assume a simple data pipeline that uses the `utcnow` function to set a variable using the SetVariable activity. + +The example can be found in the `examples/fabric/mock_example` folder. ```python - FunctionsRepository.register("concat", lambda arguments: "".join(arguments)) - FunctionsRepository.register("trim", lambda text, trim_argument: text.strip(trim_argument[0])) + +# Arrange +fabric_folder = Path(request.fspath.dirname, "fabric") +test_framework = TestFramework(framework_type=TestFrameworkType.Fabric, root_folder_path=fabric_folder) +pipeline = test_framework.get_pipeline_by_name("ExamplePipeline") + +# Mock the utcnow function to return a fixed datetime: +utcnow_mock = mock_utcnow("1990-05-12T10:30:00Z") + +# Act +activities = test_framework.evaluate_pipeline( + pipeline=pipeline, + parameters=[] + mocks=[utcnow_mock] +) + +# Assert +activity = next(activities) +assert activity.type_properties["value"].result == ["1990-05-12T10:30:00Z", "1990-05-12T10:30:00Z"] +``` + +### List of Values Mocking + +You can also provide a list of values to be returned in sequence. This is useful when you want to simulate different results for each call to the function. + +```python +def test_mock_utcnow_example(request: pytest.FixtureRequest) -> None: +# Arrange +fabric_folder = Path(request.fspath.dirname, "fabric") +test_framework = TestFramework(framework_type=TestFrameworkType.Fabric, root_folder_path=fabric_folder) +pipeline = test_framework.get_pipeline_by_name("ExamplePipeline") + +# Create a global mock for utcnow +utcnow_mock = mock_utcnow( + [ + "1980-05-12T10:30:00Z", + "1990-05-12T10:30:00Z", + ] +) + +# Act +activities = test_framework.evaluate_pipeline( + pipeline, + parameters=[], + mocks=[utcnow_mock], +) + +# Assert +activity = next(activities) +assert activity.type_properties["value"].result == ["1980-05-12T10:30:00Z", "1990-05-12T10:30:00Z"] + +# Assert that there are no more activities +with pytest.raises(StopIteration): + next(activities) +``` + +If the values are exhausted an error will be raised. This is similar to how `unittest.mock` works. + + +## Scope of Mocked Functions + +By default, the mocked result apply to all pipelines, activties, and type properties in the current test run (i.e., when the the framework evaluates the pipline or activity). +You can also specify a scope which allows you to limit the mocked result to a specific pipeline, activity, or type property to test more specific scenarios. + +```python + +# TODO: Add example code for mocking expression functions + ``` diff --git a/examples/fabric/mock_example/fabric/AdditionalExamplePipeline.DataPipeline/.platform b/examples/fabric/mock_example/fabric/AdditionalExamplePipeline.DataPipeline/.platform new file mode 100644 index 00000000..5f39bdcc --- /dev/null +++ b/examples/fabric/mock_example/fabric/AdditionalExamplePipeline.DataPipeline/.platform @@ -0,0 +1,11 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", + "metadata": { + "type": "DataPipeline", + "displayName": "AdditionalExamplePipeline" + }, + "config": { + "version": "2.0", + "logicalId": "c7986cc7-a6df-45fc-b4b6-dfc3cbddf2a6" + } +} diff --git a/examples/fabric/mock_example/fabric/AdditionalExamplePipeline.DataPipeline/pipeline-content.json b/examples/fabric/mock_example/fabric/AdditionalExamplePipeline.DataPipeline/pipeline-content.json new file mode 100644 index 00000000..74e2f1cd Binary files /dev/null and b/examples/fabric/mock_example/fabric/AdditionalExamplePipeline.DataPipeline/pipeline-content.json differ diff --git a/examples/fabric/mock_example/fabric/ExamplePipeline.DataPipeline/.platform b/examples/fabric/mock_example/fabric/ExamplePipeline.DataPipeline/.platform new file mode 100644 index 00000000..d7cdb812 --- /dev/null +++ b/examples/fabric/mock_example/fabric/ExamplePipeline.DataPipeline/.platform @@ -0,0 +1,11 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/gitIntegration/platformProperties/2.0.0/schema.json", + "metadata": { + "type": "DataPipeline", + "displayName": "ExamplePipeline" + }, + "config": { + "version": "2.0", + "logicalId": "c7986cc7-a6df-45fc-b4b6-dfc3cbddf2a6" + } +} diff --git a/examples/fabric/mock_example/fabric/ExamplePipeline.DataPipeline/pipeline-content.json b/examples/fabric/mock_example/fabric/ExamplePipeline.DataPipeline/pipeline-content.json new file mode 100644 index 00000000..079090d2 Binary files /dev/null and b/examples/fabric/mock_example/fabric/ExamplePipeline.DataPipeline/pipeline-content.json differ diff --git a/examples/fabric/mock_example/test_fabric_mock_utcnow_example.py b/examples/fabric/mock_example/test_fabric_mock_utcnow_example.py new file mode 100644 index 00000000..b4f5cce2 --- /dev/null +++ b/examples/fabric/mock_example/test_fabric_mock_utcnow_example.py @@ -0,0 +1,106 @@ +from pathlib import Path + +import pytest +from data_factory_testing_framework import TestFramework, TestFrameworkType +from data_factory_testing_framework.mock_helpers import mock_utcnow + + +def test_mock_utcnow_single_value(request: pytest.FixtureRequest) -> None: + # Arrange + fabric_folder = Path(request.fspath.dirname, "fabric") + test_framework = TestFramework(framework_type=TestFrameworkType.Fabric, root_folder_path=fabric_folder) + pipeline = test_framework.get_pipeline_by_name("ExamplePipeline") + + # Create a global mock for utcnow + utcnow_mock = mock_utcnow("1990-05-12T10:30:00Z") + + # Act + activities = test_framework.evaluate_pipeline( + pipeline, + parameters=[], + mocks=[utcnow_mock], + ) + + # Assert + activity = next(activities) + + assert activity.type_properties["value"].result == ["1990-05-12T10:30:00Z", "1990-05-12T10:30:00Z"] + + activity = next(activities) + # note: we ignore the assertions here for the purpose of this example + + # Assert that there are no more activities + with pytest.raises(StopIteration): + next(activities) + +def test_mock_utcnow_multiple_values(request: pytest.FixtureRequest) -> None: + # Arrange + fabric_folder = Path(request.fspath.dirname, "fabric") + test_framework = TestFramework(framework_type=TestFrameworkType.Fabric, root_folder_path=fabric_folder) + pipeline = test_framework.get_pipeline_by_name("ExamplePipeline") + + # Create a global mock for utcnow + utcnow_mock = mock_utcnow( + [ + "1980-05-12T10:30:00Z", + "1990-05-12T10:30:00Z", + ] + ) + + # Act + activities = test_framework.evaluate_pipeline( + pipeline, + parameters=[], + mocks=[utcnow_mock], + ) + + # Assert + activity = next(activities) + assert activity.type_properties["value"].result == ["1980-05-12T10:30:00Z", "1990-05-12T10:30:00Z"] + + activity = next(activities) + # note: we ignore the assertions here for the purpose of this example + + # Assert that there are no more activities + with pytest.raises(StopIteration): + next(activities) + +def test_mock_utcnow_with_activity_scope(request: pytest.FixtureRequest) -> None: + # Arrange + fabric_folder = Path(request.fspath.dirname, "fabric") + test_framework = TestFramework(framework_type=TestFrameworkType.Fabric, root_folder_path=fabric_folder) + pipeline = test_framework.get_pipeline_by_name("ExamplePipeline") + + # Create a mock for utcnow with activity scope + utcnow_mock = mock_utcnow("1980-05-12T10:30:00Z", activity="Set Input Data") + utcnow_mock_set_addtional_input = mock_utcnow("1990-05-12T10:30:00Z", activity="Set Additional Input Data") + + # Act + activities = test_framework.evaluate_pipeline( + pipeline, + parameters=[], + mocks=[ + utcnow_mock, + utcnow_mock_set_addtional_input + ], + ) + + # Assert + activity = next(activities) + assert activity.type_properties["value"].result == ["1980-05-12T10:30:00Z", "1980-05-12T10:30:00Z"] + + activity = next(activities) + assert activity.type_properties["value"].result == ["1990-05-12T10:30:00Z", "1990-05-12T10:30:00Z"] + + # Assert that there are no more activities + with pytest.raises(StopIteration): + next(activities) + +def test_mock_utcnow_with_scope_different_pipeline(request: pytest.FixtureRequest) -> None: + + # TODO: create a nested pipeline to test the scope functionality + # Need a nested pipeline to test the scope functionality + + pass + + diff --git a/examples/fabric/simple_web_hook/fabric/ExamplePipeline.DataPipeline/pipeline-content.json b/examples/fabric/simple_web_hook/fabric/ExamplePipeline.DataPipeline/pipeline-content.json index 70040d8d..0564796a 100644 Binary files a/examples/fabric/simple_web_hook/fabric/ExamplePipeline.DataPipeline/pipeline-content.json and b/examples/fabric/simple_web_hook/fabric/ExamplePipeline.DataPipeline/pipeline-content.json differ diff --git a/examples/fabric/simple_web_hook/test_fabric_simple_webhook_with_utcnow_mock.py b/examples/fabric/simple_web_hook/test_fabric_simple_webhook_with_utcnow_mock.py new file mode 100644 index 00000000..d8c2e92f --- /dev/null +++ b/examples/fabric/simple_web_hook/test_fabric_simple_webhook_with_utcnow_mock.py @@ -0,0 +1,32 @@ +from pathlib import Path + +import pytest +from data_factory_testing_framework import TestFramework, TestFrameworkType +from data_factory_testing_framework.mock_helpers import mock_utcnow + + +def test_simple_web_hook_with_utcnow_mock(request: pytest.FixtureRequest) -> None: + # Arrange + fabric_folder = Path(request.fspath.dirname, "fabric") + test_framework = TestFramework(framework_type=TestFrameworkType.Fabric, root_folder_path=fabric_folder) + pipeline = test_framework.get_pipeline_by_name("ExamplePipeline") + + # Add a global utcnow mock + utcnow_mock = mock_utcnow("2025-05-12T10:30:00Z") + + # Act + activities = test_framework.evaluate_pipeline(pipeline, [], mocks=[utcnow_mock]) + + # Assert + activity = next(activities) + assert activity.name == "Set Input Data" + + activity = next(activities) + assert activity.name == "Call Webhook" + + activity = next(activities) + assert activity.name == "Call Webhook" + + # Assert that there are no more activities + with pytest.raises(StopIteration): + next(activities) diff --git a/examples/fabric/simple_web_hook/test_mock_utcnow_example.py b/examples/fabric/simple_web_hook/test_mock_utcnow_example.py new file mode 100644 index 00000000..58ac099b --- /dev/null +++ b/examples/fabric/simple_web_hook/test_mock_utcnow_example.py @@ -0,0 +1,37 @@ +import pytest +from data_factory_testing_framework import TestFramework, TestFrameworkType +from data_factory_testing_framework.mock_helpers import mock_utcnow +from pathlib import Path + +def test_mock_utcnow_example(request: pytest.FixtureRequest) -> None: + # Arrange + fabric_folder = Path(request.fspath.dirname, "fabric") + test_framework = TestFramework( + framework_type=TestFrameworkType.Fabric, + root_folder_path=fabric_folder + ) + pipeline = test_framework.get_pipeline_by_name("ExamplePipeline") + + # Create a global mock for utcnow + utcnow_mock = mock_utcnow("2025-05-12T10:30:00Z") + + # Act + activities = test_framework.evaluate_pipeline( + pipeline, + parameters=[], + mocks=[utcnow_mock], + ) + + # Assert + activity = next(activities) + assert activity.name == "Set Input Data" + + activity = next(activities) + assert activity.name == "Call Webhook" + + activity = next(activities) + assert activity.name == "Call Webhook" + + # Assert that there are no more activities + with pytest.raises(StopIteration): + next(activities) diff --git a/src/data_factory_testing_framework/_expression_runtime/expression_runtime.py b/src/data_factory_testing_framework/_expression_runtime/expression_runtime.py index 5edf6836..a5f7c883 100644 --- a/src/data_factory_testing_framework/_expression_runtime/expression_runtime.py +++ b/src/data_factory_testing_framework/_expression_runtime/expression_runtime.py @@ -1,4 +1,5 @@ import re +from typing import List from data_factory_testing_framework._expression_runtime.data_factory_expression.expression_transformer import ( ExpressionTransformer as DataFactoryTestingFrameworkExpressionsTransformer, @@ -12,6 +13,8 @@ StateIterationItemNotSetError, VariableNotFoundError, ) +from data_factory_testing_framework.mock import ExpressionMock +from data_factory_testing_framework.mock_context import MockContext from data_factory_testing_framework.state import PipelineRunState, RunParameterType @@ -21,12 +24,41 @@ def __init__(self) -> None: self.dftf_expressions_transformer = DataFactoryTestingFrameworkExpressionsTransformer() self.dftf_expressions_evaluator = DataFactoryTestingFrameworkExpressionsEvaluator() - def evaluate(self, expression: str, state: PipelineRunState) -> str: + def _build_mock_config( + self, + mocks: List[ExpressionMock], + mock_context: MockContext, + ) -> dict: + """Builds a mock configuration dictionary from the provided mocks and context.""" + mock_config = {} + + for mock in mocks: + if mock.scope.is_in_scope(mock_context): + # If we already have a mock for this function, we throw an error + if mock.function_name in mock_config: + raise ValueError( + f'Duplicate mock function name detected: "{mock.function_name}". ' + "Please ensure mocks have mutually exclusive scopes." + ) + else: + mock_config[mock.function_name] = mock.mock_result + + return mock_config + + def evaluate( + self, + expression: str, + state: PipelineRunState, + mocks: List[ExpressionMock], + mock_context: MockContext, + ) -> str: + """Evaluate an expression with optional mocks and context.""" + mock_config = self._build_mock_config(mocks, mock_context) dftf_transformed_expression = self.dftf_expressions_transformer.transform_to_dftf_evaluator_expression( expression, state ) try: - result = self.dftf_expressions_evaluator.evaluate(dftf_transformed_expression, state) + result = self.dftf_expressions_evaluator.evaluate(dftf_transformed_expression, state, mock_config) except Exception as e: # match the exception type (coming from .NET) to the one we expect missing_parameter_match = re.match( diff --git a/src/data_factory_testing_framework/_pythonnet/data_factory_testing_framework_expressions_evaluator.py b/src/data_factory_testing_framework/_pythonnet/data_factory_testing_framework_expressions_evaluator.py index f592f77e..6d0f47f8 100644 --- a/src/data_factory_testing_framework/_pythonnet/data_factory_testing_framework_expressions_evaluator.py +++ b/src/data_factory_testing_framework/_pythonnet/data_factory_testing_framework_expressions_evaluator.py @@ -11,7 +11,18 @@ class DataFactoryTestingFrameworkExpressionsEvaluator: @staticmethod - def evaluate(expression: str, state: PipelineRunState) -> Union[str, int, float, bool, dict, list]: + def evaluate( + expression: str, + state: PipelineRunState, + mock_config: dict[str, Union[str, int, float, bool, dict, list]], + ) -> Union[str, int, float, bool, dict, list]: + """Evaluate an expression using the Data Factory Testing Framework's .NET Evaluator. + + Args: + expression (str): The expression to evaluate. + state (PipelineRunState): The state containing parameters, variables, and activity results. + mock_config (dict, optional): A dictionary containing mock configurations for the evaluation, mapping function names to their mock results. + """ evaluator = Evaluator() parameters = { "globalParameters": {}, @@ -58,5 +69,6 @@ def evaluate(expression: str, state: PipelineRunState) -> Union[str, int, float, json.dumps(variables), state_iter_item_json, json.dumps(activity_results), + json.dumps(mock_config), ) return json.loads(result)["result"] diff --git a/src/data_factory_testing_framework/_test_framework.py b/src/data_factory_testing_framework/_test_framework.py index ea2deb1f..abd9d277 100644 --- a/src/data_factory_testing_framework/_test_framework.py +++ b/src/data_factory_testing_framework/_test_framework.py @@ -12,6 +12,7 @@ from data_factory_testing_framework.exceptions import ( NoRemainingPipelineActivitiesMeetDependencyConditionsError, ) +from data_factory_testing_framework.mock import ExpressionMock from data_factory_testing_framework.models import Pipeline from data_factory_testing_framework.models.activities import ( Activity, @@ -98,7 +99,9 @@ def evaluate_activity(self, activity: Activity, state: PipelineRunState) -> Iter """ return self.evaluate_activities([activity], state) - def evaluate_pipeline(self, pipeline: Pipeline, parameters: List[RunParameter]) -> Iterator[Activity]: + def evaluate_pipeline( + self, pipeline: Pipeline, parameters: List[RunParameter], mocks: Optional[List["ExpressionMock"]] = None + ) -> Iterator[Activity]: """Evaluates all pipeline activities using the provided parameters. The order of activity execution is simulated based on the dependencies. @@ -107,15 +110,27 @@ def evaluate_pipeline(self, pipeline: Pipeline, parameters: List[RunParameter]) Args: pipeline: The pipeline to evaluate. parameters: The parameters to use for evaluating the pipeline. + mocks: Optional expression mocks to use during evaluation. Returns: A list of evaluated pipelines, which can be more than 1 due to possible child activities. """ + mocks = mocks or [] parameters = pipeline.validate_and_append_default_parameters(parameters) state = PipelineRunState(parameters, pipeline.get_run_variables()) - return self.evaluate_activities(pipeline.activities, state) - def evaluate_activities(self, activities: List[Activity], state: PipelineRunState) -> Iterator[Activity]: + return self._evaluate_activities( + activities=pipeline.activities, + state=state, + mocks=mocks, + ) + + def evaluate_activities( + self, + activities: List[Activity], + state: PipelineRunState, + mocks: Optional[List["ExpressionMock"]] = None, + ) -> Iterator[Activity]: """Evaluates all activities using the provided state. The order of activity execution is simulated based on the dependencies. @@ -124,10 +139,25 @@ def evaluate_activities(self, activities: List[Activity], state: PipelineRunStat Args: activities: The activities to evaluate. state: The state to use for evaluating the pipeline. + mocks: Optional expression mocks to use during evaluation. Returns: A list of evaluated pipelines, which can be more than 1 due to possible child activities. """ + mocks = mocks or [] + + return self._evaluate_activities( + activities, + state, + mocks=mocks, + ) + + def _evaluate_activities( + self, + activities: List[Activity], + state: PipelineRunState, + mocks: List[ExpressionMock], + ) -> Iterator[Activity]: fail_activity_evaluated = False while len(state.scoped_activity_results) != len(activities): any_activity_evaluated = False @@ -136,7 +166,7 @@ def evaluate_activities(self, activities: List[Activity], state: PipelineRunStat and a.are_dependency_condition_met(state), activities, ): - evaluated_activity = activity.evaluate(state) + evaluated_activity = activity.evaluate(state, mocks) if not self._is_iteration_activity(evaluated_activity) or ( isinstance(evaluated_activity, ExecutePipelineActivity) and not self.should_evaluate_child_pipelines ): @@ -168,6 +198,7 @@ def evaluate_activities(self, activities: List[Activity], state: PipelineRunStat pipeline, activity.get_child_run_parameters(state), self.evaluate_activities, + mocks, ) if not isinstance(activity, ExecutePipelineActivity) and isinstance(activity, ControlActivity): @@ -175,6 +206,7 @@ def evaluate_activities(self, activities: List[Activity], state: PipelineRunStat activities_iterator = control_activity.evaluate_control_activities( state, self.evaluate_activities, + mocks, ) for child_activity in activities_iterator: diff --git a/src/data_factory_testing_framework/mock.py b/src/data_factory_testing_framework/mock.py new file mode 100644 index 00000000..6b0a02e5 --- /dev/null +++ b/src/data_factory_testing_framework/mock.py @@ -0,0 +1,81 @@ +from typing import TYPE_CHECKING, Dict, List, Optional, Union +from typing import Any as TypingAny + +from data_factory_testing_framework.mock_context import MockContext + +if TYPE_CHECKING: + from data_factory_testing_framework.models._pipeline import Pipeline + from data_factory_testing_framework.models.activities._activity import Activity + + +class _AnyMarker(object): + """Marker class to indicate that a mock applies to any scope level.""" + + pass + + +Any = _AnyMarker() # Marker for any scope level, used in Scope class + + +class Scope: + """Defines where a mock should be applied. + + A scope can be defined at the pipeline, activity, or property level. + Use the `Any` constant to indicate a mock applies to all instances at that level. + """ + + + def __init__( + self, + pipeline: Union[str, "Pipeline", _AnyMarker] = Any, + activity: Union[str, "Activity", _AnyMarker] = Any, + type_property: Union[str, _AnyMarker] = Any, + ) -> None: + """Initialize a Scope object.""" + self.pipeline = pipeline + self.activity = activity + self.type_property = type_property + + def is_in_scope( + self, mock_context: MockContext + ) -> bool: + """Check if the mock is in scope based on the provided context.""" + pipeline_match = self.pipeline == mock_context.pipeline or ( + isinstance(self.pipeline, str) and self.pipeline == mock_context.pipeline.name + ) or self.pipeline == Any + + activity_match = ( + self.activity == mock_context.activity or + (isinstance(self.activity, str) and self.activity == mock_context.activity.name) or + self.activity == Any + ) + + type_property_match = ( + self.type_property == mock_context.property_path or + (isinstance(self.type_property, str) and self.type_property == mock_context.property_path) or + self.type_property == Any + ) + return pipeline_match and activity_match and type_property_match + + +class Mock: + """Base class for all mock implementations.""" + + def __init__(self, scope: Optional[Scope] = None) -> None: + """Initialize a mock with an optional scope.""" + self.scope = scope or Scope() + + +class ExpressionMock(Mock): + """Mocks an expression function in Data Factory.""" + + def __init__( + self, + function_name: str, + mock_result: Union[str, int, float, bool, List[TypingAny], Dict[str, TypingAny]], + scope: Optional[Scope] = None, + ) -> None: + """Initialize an ExpressionMock with a function name, mock result, and optional scope.""" + self.function_name = function_name + self.mock_result = mock_result + super().__init__(scope) diff --git a/src/data_factory_testing_framework/mock_context.py b/src/data_factory_testing_framework/mock_context.py new file mode 100644 index 00000000..18167af5 --- /dev/null +++ b/src/data_factory_testing_framework/mock_context.py @@ -0,0 +1,16 @@ + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from data_factory_testing_framework.models._pipeline import Pipeline + from data_factory_testing_framework.models.activities._activity import Activity + + + + +@dataclass +class MockContext: + pipeline: Optional["Pipeline"] = None + activity: Optional["Activity"] = None + property_path: Optional[str] = None diff --git a/src/data_factory_testing_framework/mock_helpers.py b/src/data_factory_testing_framework/mock_helpers.py new file mode 100644 index 00000000..5af2e1fc --- /dev/null +++ b/src/data_factory_testing_framework/mock_helpers.py @@ -0,0 +1,111 @@ +from typing import List, Optional, Union, overload + +from data_factory_testing_framework.mock import Any, ExpressionMock, Scope + + +@overload +def mock_utcnow(datetime_str: str) -> ExpressionMock: ... + +@overload +def mock_utcnow(datetime_str: str, scope: Scope) -> ExpressionMock: ... + +@overload +def mock_utcnow( + datetime_str: str, + *, + pipeline: Union[str, object] = None, + activity: Union[str, object] = None, + type_property: Union[str, object] = None +) -> ExpressionMock: ... + +def mock_utcnow( + datetime_str: str, + scope: Optional[Scope] = None, + *, + pipeline: Union[str, object] = None, + activity: Union[str, object] = None, + type_property: Union[str, object] = None +) -> ExpressionMock: + individual_params_provided = any(param is not None for param in [pipeline, activity, type_property]) + if scope is not None and individual_params_provided: + raise ValueError("Cannot provide both 'scope' and individual scope parameters. Use either a Scope object or individual parameters.") + if individual_params_provided: + created_scope = Scope( + pipeline=pipeline if pipeline is not None else Any, + activity=activity if activity is not None else Any, + type_property=type_property if type_property is not None else Any + ) + return ExpressionMock("utcnow", datetime_str, created_scope) + return ExpressionMock("utcnow", datetime_str, scope) + +@overload +def mock_guid(guid_values: Union[str, List[str]]) -> ExpressionMock: ... + +@overload +def mock_guid(guid_values: Union[str, List[str]], scope: Scope) -> ExpressionMock: ... + +@overload +def mock_guid( + guid_values: Union[str, List[str]], + *, + pipeline: Union[str, object] = None, + activity: Union[str, object] = None, + type_property: Union[str, object] = None +) -> ExpressionMock: ... + +def mock_guid( + guid_values: Union[str, List[str]], + scope: Optional[Scope] = None, + *, + pipeline: Union[str, object] = None, + activity: Union[str, object] = None, + type_property: Union[str, object] = None +) -> ExpressionMock: + values = [guid_values] if isinstance(guid_values, str) else guid_values + individual_params_provided = any(param is not None for param in [pipeline, activity, type_property]) + if scope is not None and individual_params_provided: + raise ValueError("Cannot provide both 'scope' and individual scope parameters. Use either a Scope object or individual parameters.") + if individual_params_provided: + created_scope = Scope( + pipeline=pipeline if pipeline is not None else Any, + activity=activity if activity is not None else Any, + type_property=type_property if type_property is not None else Any + ) + return ExpressionMock("guid", values, created_scope) + return ExpressionMock("guid", values, scope) + +@overload +def mock_rand(values: Union[int, List[int]]) -> ExpressionMock: ... + +@overload +def mock_rand(values: Union[int, List[int]], scope: Scope) -> ExpressionMock: ... + +@overload +def mock_rand( + values: Union[int, List[int]], + *, + pipeline: Union[str, object] = None, + activity: Union[str, object] = None, + type_property: Union[str, object] = None +) -> ExpressionMock: ... + +def mock_rand( + values: Union[int, List[int]], + scope: Optional[Scope] = None, + *, + pipeline: Union[str, object] = None, + activity: Union[str, object] = None, + type_property: Union[str, object] = None +) -> ExpressionMock: + values_list = [values] if isinstance(values, int) else values + individual_params_provided = any(param is not None for param in [pipeline, activity, type_property]) + if scope is not None and individual_params_provided: + raise ValueError("Cannot provide both 'scope' and individual scope parameters. Use either a Scope object or individual parameters.") + if individual_params_provided: + created_scope = Scope( + pipeline=pipeline if pipeline is not None else Any, + activity=activity if activity is not None else Any, + type_property=type_property if type_property is not None else Any + ) + return ExpressionMock("rand", values_list, created_scope) + return ExpressionMock("rand", values_list, scope) diff --git a/src/data_factory_testing_framework/models/_data_factory_element.py b/src/data_factory_testing_framework/models/_data_factory_element.py index b45441f7..b4f0060b 100644 --- a/src/data_factory_testing_framework/models/_data_factory_element.py +++ b/src/data_factory_testing_framework/models/_data_factory_element.py @@ -1,11 +1,13 @@ import json -from typing import Any +from typing import Any, List from data_factory_testing_framework._expression_runtime.expression_runtime import ExpressionRuntime from data_factory_testing_framework.exceptions import ( DataFactoryElementEvaluationError, ) from data_factory_testing_framework.exceptions._user_error import UserError +from data_factory_testing_framework.mock import ExpressionMock +from data_factory_testing_framework.mock_context import MockContext from data_factory_testing_framework.models._data_factory_object_type import DataFactoryObjectType from data_factory_testing_framework.state import RunState @@ -23,17 +25,21 @@ def __init__(self, expression: str) -> None: self.expression = expression self.result: DataFactoryObjectType = None - def evaluate(self, state: RunState) -> DataFactoryObjectType: - """Evaluate the expression.""" + def evaluate( + self, + state: RunState, + mocks: List[ExpressionMock], + mock_context: MockContext, + ) -> DataFactoryObjectType: + """Evaluates the expression and returns the result.""" try: expression_runtime = ExpressionRuntime() - self.result = expression_runtime.evaluate(self.expression, state) + self.result = expression_runtime.evaluate(self.expression, state, mocks, mock_context) return self.result except UserError as e: raise e from e except Exception as e: raise DataFactoryElementEvaluationError(f"Error evaluating expression: {self.expression}") from e - return self.result def get_json_value(self) -> Any: # noqa: ANN401 diff --git a/src/data_factory_testing_framework/models/_pipeline.py b/src/data_factory_testing_framework/models/_pipeline.py index 27527c64..fc01fbbc 100644 --- a/src/data_factory_testing_framework/models/_pipeline.py +++ b/src/data_factory_testing_framework/models/_pipeline.py @@ -29,6 +29,21 @@ def __init__( self.variables: dict = kwargs["variables"] if "variables" in kwargs else {} self.activities = activities self.annotations = kwargs["annotations"] if "annotations" in kwargs else [] + self._set_activity_pipeline_reference() + + def _set_activity_pipeline_reference(self) -> None: + """Set the pipeline reference for all activities in the pipeline.""" + all_child_activties = self.activities.copy() + + # Flatten the nested activities and set the pipeline reference + # for each activity in the pipeline. + # For control activities, we also need to set the pipeline reference + # for their nested activities. + while all_child_activties: + current_child_activity = all_child_activties.pop(0) + all_child_activties.extend(current_child_activity.nested_activities) + current_child_activity._pipeline = self + def get_activity_by_name(self, name: str) -> "Activity": """Get an activity by name. Throws an exception if the activity is not found. diff --git a/src/data_factory_testing_framework/models/activities/_activity.py b/src/data_factory_testing_framework/models/activities/_activity.py index cc72dfbf..a58136fe 100644 --- a/src/data_factory_testing_framework/models/activities/_activity.py +++ b/src/data_factory_testing_framework/models/activities/_activity.py @@ -1,11 +1,16 @@ -from typing import Any, List, Optional +from typing import TYPE_CHECKING, Any, List, Optional -from data_factory_testing_framework.models import DataFactoryElement +from data_factory_testing_framework.mock import ExpressionMock +from data_factory_testing_framework.mock_context import MockContext +from data_factory_testing_framework.models._data_factory_element import DataFactoryElement from data_factory_testing_framework.models.activities._activity_dependency import ( ActivityDependency, ) from data_factory_testing_framework.state import DependencyCondition, PipelineRunState +if TYPE_CHECKING: + from data_factory_testing_framework.models._pipeline import Pipeline + class Activity: def __init__(self, name: str, type: str, policy: Optional[dict] = None, **kwargs: Any) -> None: # noqa: ANN401, A002 @@ -34,9 +39,39 @@ def __init__(self, name: str, type: str, policy: Optional[dict] = None, **kwargs self.status: DependencyCondition = None self.output = {} + self._pipeline: Optional["Pipeline"] = None + + @property + def pipeline(self) -> "Pipeline": + """Get the pipeline that contains this activity.""" + if self._pipeline is None: + raise ValueError("Pipeline is not set for this activity.") + return self._pipeline + - def evaluate(self, state: PipelineRunState) -> "Activity": - self._evaluate_expressions(self, state, types_to_ignore=[Activity]) + @property + def nested_activities(self) -> List["Activity"]: + """Get the nested activities of this activity. + + Some activties, like IfConditionActivity or SwitchActivity, have nested activities. + Other activities, like SetVariableActivity or FilterActivity, do not have nested activities and return an empty list. + """ + return [] + + def evaluate( + self, + state: PipelineRunState, + mocks: Optional[List[ExpressionMock]] = None, + ) -> "Activity": + """Evaluate the activity expressions and set the status to Succeeded.""" + mocks = mocks or [] + + self._evaluate_expressions( + self, + state, + mocks, + types_to_ignore=[Activity], + ) self.status = DependencyCondition.Succeeded self.output = {} return self @@ -64,8 +99,10 @@ def _evaluate_expressions( self, obj: Any, # noqa: ANN401 state: PipelineRunState, - visited: Optional[List[Any]] = None, # noqa: ANN401 - types_to_ignore: Optional[List[Any]] = None, # noqa: ANN401 + mocks: List[ExpressionMock], + visited: Optional[List[Any]] = None, + types_to_ignore: Optional[List[Any]] = None, + property_path: Optional[str] = None, ) -> None: if visited is None: visited = [] @@ -76,7 +113,16 @@ def _evaluate_expressions( visited.append(obj) if data_factory_element := isinstance(obj, DataFactoryElement) and obj: - data_factory_element.evaluate(state) + # TODO: clarify how we build the property path for DataFactoryElement + data_factory_element.evaluate( + state=state, + mocks=mocks, + mock_context=MockContext( + pipeline=self._pipeline, + activity=self, + property_path=property_path + ), + ) return # Attributes @@ -92,8 +138,14 @@ def _evaluate_expressions( attribute = getattr(obj, attribute_name) if attribute is None: continue - - self._evaluate_expressions(attribute, state, visited, types_to_ignore) + self._evaluate_expressions( + obj=attribute, + state=state, + mocks=mocks, + visited=visited, + types_to_ignore=types_to_ignore, + property_path=f"{property_path}.{attribute_name}" if property_path else attribute_name, + ) # Dictionary if isinstance(obj, dict): @@ -101,11 +153,18 @@ def _evaluate_expressions( if "activities" in key: continue - self._evaluate_expressions(obj[key], state, visited, types_to_ignore) + self._evaluate_expressions( + obj=obj[key], + state=state, + mocks=mocks, + visited=visited, + types_to_ignore=types_to_ignore, + property_path=f"{property_path}.{key}" if property_path else key, + ) # List if isinstance(obj, list): - for item in obj: + for index, item in enumerate(obj): ignore_item = False for type_to_ignore in types_to_ignore: if isinstance(item, type_to_ignore): @@ -114,7 +173,14 @@ def _evaluate_expressions( if ignore_item: continue - self._evaluate_expressions(item, state, visited, types_to_ignore) + self._evaluate_expressions( + obj=item, + state=state, + mocks=mocks, + visited=visited, + types_to_ignore=types_to_ignore, + property_path=f"{property_path}[{index}]" if property_path else f"[{index}]", + ) def set_result(self, result: DependencyCondition, output: Optional[Any] = None) -> None: # noqa: ANN401 self.status = result diff --git a/src/data_factory_testing_framework/models/activities/_activity_dependency.py b/src/data_factory_testing_framework/models/activities/_activity_dependency.py index 135c60fc..4032e84e 100644 --- a/src/data_factory_testing_framework/models/activities/_activity_dependency.py +++ b/src/data_factory_testing_framework/models/activities/_activity_dependency.py @@ -1,6 +1,6 @@ from typing import List, Union -from data_factory_testing_framework.state import DependencyCondition +from data_factory_testing_framework.state._dependency_condition import DependencyCondition class ActivityDependency: diff --git a/src/data_factory_testing_framework/models/activities/_append_variable_activity.py b/src/data_factory_testing_framework/models/activities/_append_variable_activity.py index 3f8884cd..f891c9d8 100644 --- a/src/data_factory_testing_framework/models/activities/_append_variable_activity.py +++ b/src/data_factory_testing_framework/models/activities/_append_variable_activity.py @@ -1,5 +1,7 @@ -from typing import Any +from typing import Any, List, Optional +from data_factory_testing_framework.mock import ExpressionMock +from data_factory_testing_framework.mock_context import MockContext from data_factory_testing_framework.models._data_factory_element import DataFactoryElement from data_factory_testing_framework.models.activities._control_activity import ControlActivity from data_factory_testing_framework.state import PipelineRunState @@ -14,16 +16,22 @@ def __init__(self, **kwargs: Any) -> None: # noqa: ANN401 """ kwargs["type"] = "AppendVariable" - super(ControlActivity, self).__init__(**kwargs) + super().__init__(**kwargs) self.variable_name: str = self.type_properties["variableName"] self.value: DataFactoryElement = self.type_properties["value"] - def evaluate(self, state: PipelineRunState) -> "AppendVariableActivity": - super(ControlActivity, self).evaluate(state) + def evaluate( + self, state: PipelineRunState, mocks: Optional[List[ExpressionMock]] = None + ) -> "AppendVariableActivity": + mocks = mocks or [] + + super().evaluate(state, mocks) if isinstance(self.value, DataFactoryElement): - evaluated_value = self.value.evaluate(state) + evaluated_value = self.value.evaluate( + state, mocks, mock_context=MockContext(pipeline=self.pipeline, activity=self, property_path="value") + ) else: evaluated_value = self.value diff --git a/src/data_factory_testing_framework/models/activities/_control_activity.py b/src/data_factory_testing_framework/models/activities/_control_activity.py index c9faa6b4..e62523ae 100644 --- a/src/data_factory_testing_framework/models/activities/_control_activity.py +++ b/src/data_factory_testing_framework/models/activities/_control_activity.py @@ -1,21 +1,25 @@ -from typing import Any, Callable, Iterator, List +from typing import Any, Callable, Iterator, List, Optional -from data_factory_testing_framework.models.activities import Activity +from data_factory_testing_framework.mock import ExpressionMock +from data_factory_testing_framework.models.activities._activity import Activity from data_factory_testing_framework.state import PipelineRunState class ControlActivity(Activity): + """This is the base class for all control activities in the pipeline.""" + def __init__(self, **kwargs: Any) -> None: # noqa: ANN401 """This is the base class for all control activities in the pipeline. Args: **kwargs: ControlActivity properties coming directly from the json representation of the activity. """ - super(Activity, self).__init__(**kwargs) + super().__init__(**kwargs) def evaluate_control_activities( self, state: PipelineRunState, evaluate_activities: Callable[[List[Activity], PipelineRunState], Iterator[Activity]], + mocks: Optional[List[ExpressionMock]] = None, ) -> Iterator[Activity]: yield from list() diff --git a/src/data_factory_testing_framework/models/activities/_execute_pipeline_activity.py b/src/data_factory_testing_framework/models/activities/_execute_pipeline_activity.py index c7b8ece1..471dc7bc 100644 --- a/src/data_factory_testing_framework/models/activities/_execute_pipeline_activity.py +++ b/src/data_factory_testing_framework/models/activities/_execute_pipeline_activity.py @@ -1,5 +1,6 @@ -from typing import Any, Callable, Iterator, List +from typing import Any, Callable, Iterator, List, Optional +from data_factory_testing_framework.mock import ExpressionMock from data_factory_testing_framework.models._data_factory_element import DataFactoryElement from data_factory_testing_framework.models._pipeline import Pipeline from data_factory_testing_framework.models.activities._activity import Activity @@ -16,7 +17,7 @@ def __init__(self, **kwargs: Any) -> None: # noqa: ANN401 """ kwargs["type"] = "ExecutePipeline" - super(ControlActivity, self).__init__(**kwargs) + super().__init__(**kwargs) self.parameters: dict = {} if "parameters" in self.type_properties: @@ -41,10 +42,12 @@ def evaluate_pipeline( pipeline: Pipeline, parameters: List[RunParameter], evaluate_activities: Callable[[List[Activity], PipelineRunState], Iterator[Activity]], + mocks: Optional[List[ExpressionMock]] = None ) -> Iterator[Activity]: + mocks = mocks or [] parameters = pipeline.validate_and_append_default_parameters(parameters) scoped_state = PipelineRunState(parameters, pipeline.get_run_variables()) - for activity in evaluate_activities(pipeline.activities, scoped_state): + for activity in evaluate_activities(pipeline.activities, scoped_state, mocks): yield activity # Set the pipelineReturnValues as evaluated by SetVariable activities to the ExecutePipelineActivity output diff --git a/src/data_factory_testing_framework/models/activities/_fail_activity.py b/src/data_factory_testing_framework/models/activities/_fail_activity.py index 08915042..73cb412d 100644 --- a/src/data_factory_testing_framework/models/activities/_fail_activity.py +++ b/src/data_factory_testing_framework/models/activities/_fail_activity.py @@ -1,6 +1,7 @@ -from typing import Any +from typing import Any, List, Optional -from data_factory_testing_framework.models.activities import ControlActivity +from data_factory_testing_framework.mock import ExpressionMock +from data_factory_testing_framework.models.activities._control_activity import ControlActivity from data_factory_testing_framework.state import DependencyCondition, PipelineRunState @@ -15,8 +16,13 @@ def __init__(self, **kwargs: Any) -> None: # noqa: ANN401 super(ControlActivity, self).__init__(**kwargs) - def evaluate(self, state: PipelineRunState) -> "FailActivity": - super(ControlActivity, self).evaluate(state) + def evaluate( + self, + state: PipelineRunState, + mocks: Optional[List[ExpressionMock]] = None + ) -> "FailActivity": + mocks = mocks or [] + super().evaluate(state, mocks) self.set_result(DependencyCondition.FAILED) diff --git a/src/data_factory_testing_framework/models/activities/_filter_activity.py b/src/data_factory_testing_framework/models/activities/_filter_activity.py index 2ebd2a9a..01d656b1 100644 --- a/src/data_factory_testing_framework/models/activities/_filter_activity.py +++ b/src/data_factory_testing_framework/models/activities/_filter_activity.py @@ -1,10 +1,12 @@ -from typing import Any +from typing import Any, List, Optional from data_factory_testing_framework.exceptions._control_activity_expression_evaluated_not_to_expected_type import ( ControlActivityExpressionEvaluatedNotToExpectedTypeError, ) +from data_factory_testing_framework.mock import ExpressionMock +from data_factory_testing_framework.mock_context import MockContext from data_factory_testing_framework.models._data_factory_element import DataFactoryElement -from data_factory_testing_framework.models.activities import ControlActivity +from data_factory_testing_framework.models.activities._control_activity import ControlActivity from data_factory_testing_framework.state import DependencyCondition, PipelineRunState @@ -20,20 +22,38 @@ def __init__( """ kwargs["type"] = "Filter" - super(ControlActivity, self).__init__(**kwargs) + super().__init__(**kwargs) self.items: DataFactoryElement = self.type_properties["items"] self.condition: DataFactoryElement = self.type_properties["condition"] - def evaluate(self, state: PipelineRunState) -> "FilterActivity": - items = self.items.evaluate(state) + def evaluate( + self, + state: PipelineRunState, + mocks: Optional[List[ExpressionMock]] = None + ) -> "FilterActivity": + mocks = mocks or [] + mock_context = MockContext( + pipeline=self.pipeline, + activity=self, + property_path="items" + ) + items = self.items.evaluate(state, mocks=mocks, mock_context=mock_context) if not isinstance(items, list): raise ControlActivityExpressionEvaluatedNotToExpectedTypeError(self.name, list) value = [] for item in items: scoped_state = state.create_iteration_scope(item) - if self.condition.evaluate(scoped_state): + if self.condition.evaluate( + scoped_state, + mocks, + mock_context=MockContext( + pipeline=self.pipeline, + activity=self, + property_path="condition", + ) + ): value.append(item) self.set_result(DependencyCondition.SUCCEEDED, {"value": value}) diff --git a/src/data_factory_testing_framework/models/activities/_for_each_activity.py b/src/data_factory_testing_framework/models/activities/_for_each_activity.py index 1d0aad61..4fea8348 100644 --- a/src/data_factory_testing_framework/models/activities/_for_each_activity.py +++ b/src/data_factory_testing_framework/models/activities/_for_each_activity.py @@ -1,10 +1,13 @@ -from typing import Any, Callable, Iterator, List +from typing import Any, Callable, Iterator, List, Optional from data_factory_testing_framework.exceptions._control_activity_expression_evaluated_not_to_expected_type import ( ControlActivityExpressionEvaluatedNotToExpectedTypeError, ) +from data_factory_testing_framework.mock import ExpressionMock +from data_factory_testing_framework.mock_context import MockContext from data_factory_testing_framework.models._data_factory_element import DataFactoryElement -from data_factory_testing_framework.models.activities import Activity, ControlActivity +from data_factory_testing_framework.models.activities._activity import Activity +from data_factory_testing_framework.models.activities._control_activity import ControlActivity from data_factory_testing_framework.state import PipelineRunState @@ -22,17 +25,28 @@ def __init__( """ kwargs["type"] = "ForEach" - super(ControlActivity, self).__init__(**kwargs) + super().__init__(**kwargs) self.activities = activities self.items: DataFactoryElement = self.type_properties["items"] - def evaluate(self, state: PipelineRunState) -> "ForEachActivity": - items = self.items.evaluate(state) + @property + def nested_activities(self) -> List["Activity"]: + """Get the nested activities of this activity.""" + return self.activities + + def evaluate( + self, + state: PipelineRunState, + mocks: Optional[List[ExpressionMock]] = None + ) -> "ForEachActivity": + mocks = mocks or [] + items = self.items.evaluate(state, mocks=mocks, mock_context=MockContext(pipeline=self.pipeline, activity=self, property_path="items")) + if not isinstance(items, list): raise ControlActivityExpressionEvaluatedNotToExpectedTypeError(self.name, list) - super(ControlActivity, self).evaluate(state) + super().evaluate(state, mocks) return self @@ -40,10 +54,12 @@ def evaluate_control_activities( self, state: PipelineRunState, evaluate_activities: Callable[[List[Activity], PipelineRunState], Iterator[Activity]], + mocks: Optional[List[ExpressionMock]] = None ) -> Iterator[Activity]: + mocks = mocks or [] for item in self.items.result: scoped_state = state.create_iteration_scope(item) - for activity in evaluate_activities(self.activities, scoped_state): + for activity in evaluate_activities(self.activities, scoped_state, mocks): yield activity state.add_scoped_activity_results_from_scoped_state(scoped_state) diff --git a/src/data_factory_testing_framework/models/activities/_if_condition_activity.py b/src/data_factory_testing_framework/models/activities/_if_condition_activity.py index d85844c8..29654ca2 100644 --- a/src/data_factory_testing_framework/models/activities/_if_condition_activity.py +++ b/src/data_factory_testing_framework/models/activities/_if_condition_activity.py @@ -1,10 +1,13 @@ -from typing import Any, Callable, Iterator, List +from typing import Any, Callable, Iterator, List, Optional from data_factory_testing_framework.exceptions._control_activity_expression_evaluated_not_to_expected_type import ( ControlActivityExpressionEvaluatedNotToExpectedTypeError, ) +from data_factory_testing_framework.mock import ExpressionMock +from data_factory_testing_framework.mock_context import MockContext from data_factory_testing_framework.models._data_factory_element import DataFactoryElement -from data_factory_testing_framework.models.activities import Activity, ControlActivity +from data_factory_testing_framework.models.activities._activity import Activity +from data_factory_testing_framework.models.activities._control_activity import ControlActivity from data_factory_testing_framework.state import PipelineRunState @@ -30,12 +33,26 @@ def __init__( self.if_false_activities = if_false_activities self.expression: DataFactoryElement = self.type_properties["expression"] - def evaluate(self, state: PipelineRunState) -> "IfConditionActivity": - evaluated_expression = self.expression.evaluate(state) + @property + def nested_activities(self) -> List["Activity"]: + """Get the nested activities of this activity.""" + return self.if_true_activities + self.if_false_activities + + def evaluate( + self, + state: PipelineRunState, + mocks: Optional[List[ExpressionMock]] = None, + ) -> "IfConditionActivity": + mocks = mocks or [] + evaluated_expression = self.expression.evaluate(state, mocks, MockContext( + pipeline=self.pipeline, + activity=self, + property_path="expression" + )) if not isinstance(evaluated_expression, bool): raise ControlActivityExpressionEvaluatedNotToExpectedTypeError(self.name, bool) - super(ControlActivity, self).evaluate(state) + super().evaluate(state, mocks) return self @@ -43,10 +60,11 @@ def evaluate_control_activities( self, state: PipelineRunState, evaluate_activities: Callable[[List[Activity], PipelineRunState], Iterator[Activity]], + mocks: Optional[List[ExpressionMock]] = None, ) -> Iterator[Activity]: scoped_state = state.create_iteration_scope() activities = self.if_true_activities if self.expression.result else self.if_false_activities - for activity in evaluate_activities(activities, scoped_state): + for activity in evaluate_activities(activities, scoped_state, mocks): yield activity state.add_scoped_activity_results_from_scoped_state(scoped_state) diff --git a/src/data_factory_testing_framework/models/activities/_set_variable_activity.py b/src/data_factory_testing_framework/models/activities/_set_variable_activity.py index 630ff8c9..3fde2fe8 100644 --- a/src/data_factory_testing_framework/models/activities/_set_variable_activity.py +++ b/src/data_factory_testing_framework/models/activities/_set_variable_activity.py @@ -1,7 +1,9 @@ -from typing import Any +from typing import Any, List, Optional +from data_factory_testing_framework.mock import ExpressionMock +from data_factory_testing_framework.mock_context import MockContext from data_factory_testing_framework.models._data_factory_element import DataFactoryElement -from data_factory_testing_framework.models.activities import ControlActivity +from data_factory_testing_framework.models.activities._control_activity import ControlActivity from data_factory_testing_framework.state import PipelineRunState @@ -14,19 +16,34 @@ def __init__(self, **kwargs: Any) -> None: # noqa: ANN401 """ kwargs["type"] = "SetVariable" - super(ControlActivity, self).__init__(**kwargs) + super().__init__(**kwargs) self.variable_name: str = self.type_properties["variableName"] self.value: DataFactoryElement = self.type_properties["value"] - def evaluate(self, state: PipelineRunState) -> "SetVariableActivity": - super(ControlActivity, self).evaluate(state) + def evaluate( + self, + state: PipelineRunState, + mocks: Optional[List[ExpressionMock]] = None + ) -> "SetVariableActivity": + mocks = mocks or [] + super().evaluate(state, mocks) + if self.type_properties["variableName"] == "pipelineReturnValue": for return_value in self.type_properties["value"]: value = return_value["value"] if isinstance(value, DataFactoryElement): - evaluated_value = value.evaluate(state) + evaluated_value = value.evaluate( + state, + mocks, + mock_context=MockContext( + pipeline=self.pipeline, + activity=self, + # TODO: Clarify what the property path should be here. + property_path=f"pipelineReturnValue.{return_value['key']}" + ) + ) else: evaluated_value = value @@ -35,7 +52,15 @@ def evaluate(self, state: PipelineRunState) -> "SetVariableActivity": return self if isinstance(self.value, DataFactoryElement): - evaluated_value = self.value.evaluate(state) + evaluated_value = self.value.evaluate( + state=state, + mocks=mocks, + mock_context=MockContext( + pipeline=self.pipeline, + activity=self, + property_path="value" + ) + ) else: evaluated_value = self.value diff --git a/src/data_factory_testing_framework/models/activities/_switch_activity.py b/src/data_factory_testing_framework/models/activities/_switch_activity.py index 8d9c4a20..a4fa4934 100644 --- a/src/data_factory_testing_framework/models/activities/_switch_activity.py +++ b/src/data_factory_testing_framework/models/activities/_switch_activity.py @@ -1,10 +1,13 @@ -from typing import Any, Callable, Dict, Generator, Iterator, List +from typing import Any, Callable, Dict, Generator, Iterator, List, Optional from data_factory_testing_framework.exceptions._control_activity_expression_evaluated_not_to_expected_type import ( ControlActivityExpressionEvaluatedNotToExpectedTypeError, ) +from data_factory_testing_framework.mock import ExpressionMock +from data_factory_testing_framework.mock_context import MockContext from data_factory_testing_framework.models._data_factory_element import DataFactoryElement -from data_factory_testing_framework.models.activities import Activity, ControlActivity +from data_factory_testing_framework.models.activities._activity import Activity +from data_factory_testing_framework.models.activities._control_activity import ControlActivity from data_factory_testing_framework.state import PipelineRunState @@ -30,12 +33,27 @@ def __init__( self.cases_activities = cases_activities self.on: DataFactoryElement = self.type_properties["on"] - def evaluate(self, state: PipelineRunState) -> "SwitchActivity": - evaluated_on = self.on.evaluate(state) + + @property + def nested_activities(self) -> List["Activity"]: + """Get the nested activities of this activity.""" + return self.default_activities + [activity for activities in self.cases_activities.values() for activity in activities] + + def evaluate( + self, + state: PipelineRunState, + mocks: Optional[List[ExpressionMock]] = None, + ) -> "SwitchActivity": + mocks = mocks or [] + evaluated_on = self.on.evaluate(state, mocks, mock_context=MockContext( + pipeline=self.pipeline, + activity=self, + property_path="on" + )) if not isinstance(evaluated_on, str): raise ControlActivityExpressionEvaluatedNotToExpectedTypeError(self.name, str) - super(ControlActivity, self).evaluate(state) + super().evaluate(state, mocks) return self @@ -43,20 +61,23 @@ def evaluate_control_activities( self, state: PipelineRunState, evaluate_activities: Callable[[List[Activity], PipelineRunState], Iterator[Activity]], + mocks: List[ExpressionMock], ) -> Iterator[Activity]: for case, activities in self.cases_activities.items(): if case == self.on.result: - return self._run_activities_in_scope(state, activities, evaluate_activities) + return self._run_activities_in_scope(state, activities, evaluate_activities, mocks) - return self._run_activities_in_scope(state, self.default_activities, evaluate_activities) + return self._run_activities_in_scope(state, self.default_activities, evaluate_activities, mocks) @staticmethod def _run_activities_in_scope( state: PipelineRunState, activities: List[Activity], evaluate_activities: Callable[[List[Activity], PipelineRunState], Iterator[Activity]], + mocks: Optional[List[ExpressionMock]] = None, ) -> Generator[Activity, None, None]: + mocks = mocks or [] scoped_state = state.create_iteration_scope() - for activity in evaluate_activities(activities, scoped_state): + for activity in evaluate_activities(activities, scoped_state, mocks): yield activity state.add_scoped_activity_results_from_scoped_state(scoped_state) diff --git a/src/data_factory_testing_framework/models/activities/_until_activity.py b/src/data_factory_testing_framework/models/activities/_until_activity.py index 314a88f6..402b51f1 100644 --- a/src/data_factory_testing_framework/models/activities/_until_activity.py +++ b/src/data_factory_testing_framework/models/activities/_until_activity.py @@ -1,10 +1,13 @@ -from typing import Any, Callable, Iterator, List +from typing import Any, Callable, Iterator, List, Optional from data_factory_testing_framework.exceptions._control_activity_expression_evaluated_not_to_expected_type import ( ControlActivityExpressionEvaluatedNotToExpectedTypeError, ) +from data_factory_testing_framework.mock import ExpressionMock +from data_factory_testing_framework.mock_context import MockContext from data_factory_testing_framework.models._data_factory_element import DataFactoryElement -from data_factory_testing_framework.models.activities import Activity, ControlActivity +from data_factory_testing_framework.models.activities._activity import Activity +from data_factory_testing_framework.models.activities._control_activity import ControlActivity from data_factory_testing_framework.state import DependencyCondition, PipelineRunState @@ -22,12 +25,17 @@ def __init__( """ kwargs["type"] = "Until" - super(ControlActivity, self).__init__(**kwargs) + super().__init__(**kwargs) self.expression: DataFactoryElement = self.type_properties["expression"] self.activities = activities - def evaluate(self, state: PipelineRunState) -> "UntilActivity": + @property + def nested_activities(self) -> List["Activity"]: + """Get the nested activities of this activity.""" + return self.activities + + def evaluate(self, state: PipelineRunState, mocks: Optional[List[ExpressionMock]] = None) -> "UntilActivity": # Explicitly not evaluate here, but in the evaluate_control_activities method after the first iteration return self @@ -35,15 +43,20 @@ def evaluate_control_activities( self, state: PipelineRunState, evaluate_activities: Callable[[List[Activity], PipelineRunState], Iterator[Activity]], + mocks: Optional[List[ExpressionMock]] = None, ) -> Iterator[Activity]: + mocks = mocks or [] while True: scoped_state = state.create_iteration_scope() - for activity in evaluate_activities(self.activities, scoped_state): + for activity in evaluate_activities(self.activities, scoped_state, mocks): yield activity state.add_scoped_activity_results_from_scoped_state(scoped_state) - evaluated_expression = self.expression.evaluate(state) + evaluated_expression = self.expression.evaluate(state, mocks=[], mock_context=MockContext( + pipeline=self._pipeline, + activity=self, + )) if not isinstance(evaluated_expression, bool): raise ControlActivityExpressionEvaluatedNotToExpectedTypeError(self.name, bool) diff --git a/tests/functional/api/classes/test_test_framework_api.py b/tests/functional/api/classes/test_test_framework_api.py index d713e89d..fc193206 100644 --- a/tests/functional/api/classes/test_test_framework_api.py +++ b/tests/functional/api/classes/test_test_framework_api.py @@ -1,8 +1,9 @@ +from ast import List import inspect import types import typing -from typing import Optional - +from typing import Optional, Union +from typing import ForwardRef from data_factory_testing_framework import TestFrameworkType from data_factory_testing_framework.models import Pipeline from data_factory_testing_framework.models.activities import Activity @@ -103,6 +104,12 @@ def test_test_framework_method_signatures() -> None: inspect.Parameter( name="state", kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=PipelineRunState ), + inspect.Parameter( + name="mocks", + kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, + default=None, + annotation=typing.Optional[typing.List[ForwardRef('ExpressionMock')]], + ), ], return_annotation=typing.Iterator[Activity], ), @@ -125,6 +132,12 @@ def test_test_framework_method_signatures() -> None: kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=typing.List[RunParameter], ), + inspect.Parameter( + name="mocks", + kind=inspect.Parameter.POSITIONAL_OR_KEYWORD, + default=None, + annotation=typing.Optional[typing.List[ForwardRef('ExpressionMock')]], + ), ], return_annotation=typing.Iterator[Activity], ), diff --git a/tests/functional/api/test_package_api.py b/tests/functional/api/test_package_api.py index de3f3fe9..96b1a75e 100644 --- a/tests/functional/api/test_package_api.py +++ b/tests/functional/api/test_package_api.py @@ -21,8 +21,7 @@ def test_package_modules() -> None: public_modules = get_public_members(package, predicate=is_public_module) # Assert - assert len(public_modules) == 3 - assert public_modules == ["exceptions", "models", "state"] + assert public_modules == ['exceptions', 'mock', 'mock_context', 'mock_helpers', 'models', 'state'] def test_package_classes() -> None: diff --git a/tests/functional/datafactory_element/test_evaluate_datafactory_element.py b/tests/functional/datafactory_element/test_evaluate_datafactory_element.py index 75593495..fc64cf4d 100644 --- a/tests/functional/datafactory_element/test_evaluate_datafactory_element.py +++ b/tests/functional/datafactory_element/test_evaluate_datafactory_element.py @@ -4,6 +4,7 @@ ParameterNotFoundError, ) from data_factory_testing_framework.exceptions._user_error import UserError +from data_factory_testing_framework.mock_context import MockContext from data_factory_testing_framework.models import DataFactoryElement from data_factory_testing_framework.state import PipelineRunState @@ -15,7 +16,7 @@ def test_evaluate_datafactory_element() -> None: data_factory_element = DataFactoryElement(expression) # Act - result = data_factory_element.evaluate(state) + result = data_factory_element.evaluate(state, mocks=[], mock_context=MockContext()) # Assert assert result == 6 @@ -29,7 +30,7 @@ def test_evaluate_datafactory_element_passes_user_error_through() -> None: # Act with pytest.raises(UserError) as e: - data_factory_element.evaluate(state) + data_factory_element.evaluate(state, mocks=[], mock_context=MockContext()) # Assert assert isinstance(e.value, ParameterNotFoundError) @@ -43,4 +44,4 @@ def test_evaluate_datafactory_element_raises_technical_errors() -> None: # Act with pytest.raises(DataFactoryElementEvaluationError): - data_factory_element.evaluate(state) + data_factory_element.evaluate(state, mocks=[], mock_context=MockContext()) diff --git a/tests/unit/functions/test_data_factory_testing_framework_expression_evaluator.py b/tests/unit/functions/test_data_factory_testing_framework_expression_evaluator.py index 08f59bd8..129ca51a 100644 --- a/tests/unit/functions/test_data_factory_testing_framework_expression_evaluator.py +++ b/tests/unit/functions/test_data_factory_testing_framework_expression_evaluator.py @@ -14,6 +14,7 @@ StateIterationItemNotSetError, VariableNotFoundError, ) +from data_factory_testing_framework.mock_context import MockContext from data_factory_testing_framework.state import ( ActivityResult, DependencyCondition, @@ -427,7 +428,7 @@ def test_evaluate( # Act evaluating the expression dftf_evaluator = DataFactoryTestingFrameworkExpressionsEvaluator() - actual = dftf_evaluator.evaluate(dftf_evaluator_expression, state) + actual = dftf_evaluator.evaluate(dftf_evaluator_expression, state, mock_config={}) # Assert evaluation assert actual == expected_evaluation @@ -440,7 +441,7 @@ def test_evaluate_function_names_are_case_insensitive() -> None: state = PipelineRunState() # Act - evaluated_value = expression_runtime.evaluate(expression, state) + evaluated_value = expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert evaluated_value == "ab" @@ -463,7 +464,7 @@ def test_evaluate_function_with_null_conditional_operator() -> None: ) # Act - evaluated_value = expression_runtime.evaluate(expression, state) + evaluated_value = expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert evaluated_value == "value1" @@ -486,7 +487,7 @@ def test_evaluate_function_with_null_conditional_operator_and_null_value() -> No ) # Act - evaluated_value = expression_runtime.evaluate(expression, state) + evaluated_value = expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert evaluated_value is None @@ -499,7 +500,7 @@ def test_evaluate_function_with_null_conditional_operator_and_system_variable() state = PipelineRunState() # Act - evaluated_value = expression_runtime.evaluate(expression, state) + evaluated_value = expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert evaluated_value is None @@ -524,7 +525,7 @@ def test_evaluate_parameter_with_complex_object_and_array_index() -> None: ) # Act - evaluated_value = expression_runtime.evaluate(expression, state) + evaluated_value = expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert evaluated_value == "value1" @@ -538,7 +539,7 @@ def test_evaluate_raises_exception_when_pipeline_parameter_not_found() -> None: # Act with pytest.raises(ParameterNotFoundError) as exinfo: - expression_runtime.evaluate(expression, state) + expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert str(exinfo.value) == "Parameter: 'parameter' of type 'RunParameterType.Pipeline' not found" @@ -552,7 +553,7 @@ def test_evaluate_raises_exception_when_pipeline_global_parameter_not_found() -> # Act with pytest.raises(ParameterNotFoundError) as exinfo: - expression_runtime.evaluate(expression, state) + expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert str(exinfo.value) == "Parameter: 'parameter' of type 'RunParameterType.Global' not found" @@ -576,7 +577,7 @@ def test_evaluate_variable_with_complex_object_and_array_index() -> None: ) # Act - evaluated_value = expression_runtime.evaluate(expression, state) + evaluated_value = expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert evaluated_value == "value1" @@ -590,7 +591,7 @@ def test_evaluate_raises_exception_when_variable_not_found() -> None: # Act with pytest.raises(VariableNotFoundError) as exinfo: - expression_runtime.evaluate(expression, state) + expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert str(exinfo.value) == "Variable 'variable' not found" @@ -615,7 +616,7 @@ def test_evaluate_dataset_with_complex_object_and_array_index() -> None: ) # Act - evaluated_value = expression_runtime.evaluate(expression, state) + evaluated_value = expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert evaluated_value == "value1" @@ -629,7 +630,7 @@ def test_evaluate_raises_exception_when_dataset_not_found() -> None: # Act with pytest.raises(ParameterNotFoundError) as exinfo: - expression_runtime.evaluate(expression, state) + expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert str(exinfo.value) == "Parameter: 'parameterName' of type 'RunParameterType.Dataset' not found" @@ -654,7 +655,7 @@ def test_evaluate_linked_service_with_complex_object_and_array_index() -> None: ) # Act - evaluated_value = expression_runtime.evaluate(expression, state) + evaluated_value = expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert evaluated_value == "value1" @@ -668,7 +669,7 @@ def test_evaluate_raises_exception_when_linked_service_not_found() -> None: # Act with pytest.raises(ParameterNotFoundError) as exinfo: - expression_runtime.evaluate(expression, state) + expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert str(exinfo.value) == "Parameter: 'parameterName' of type 'RunParameterType.LinkedService' not found" @@ -682,7 +683,7 @@ def test_evaluate_raises_exception_when_activity_not_found() -> None: # Act with pytest.raises(ActivityNotFoundError) as exinfo: - expression_runtime.evaluate(expression, state) + expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert str(exinfo.value) == "Activity with name 'activityName' not found" @@ -696,7 +697,7 @@ def test_evaluate_raises_exception_when_state_iteration_item_not_set() -> None: # Act with pytest.raises(StateIterationItemNotSetError) as exinfo: - expression_runtime.evaluate(expression, state) + expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert str(exinfo.value) == "Iteration item not set." @@ -713,7 +714,7 @@ def test_evaluate_complex_item() -> None: ) # Act - evaluated_value = expression_runtime.evaluate(expression, state) + evaluated_value = expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert evaluated_value == "value1" @@ -730,7 +731,7 @@ def test_evaluate_system_variable() -> None: ) # Act - actual = expression_runtime.evaluate(expression, state) + actual = expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert actual == "123" @@ -744,7 +745,7 @@ def test_evaluate_system_variable_raises_exception_when_parameter_not_set() -> N # Act with pytest.raises(ParameterNotFoundError) as exinfo: - expression_runtime.evaluate(expression, state) + expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert str(exinfo.value) == "Parameter: 'RunId' of type 'RunParameterType.System' not found" @@ -761,7 +762,7 @@ def test_evaluate_library_variable() -> None: ) # Act - evaluated_value = expression_runtime.evaluate(expression, state) + evaluated_value = expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert evaluated_value == "value" @@ -775,7 +776,7 @@ def test_evaluate_library_variable_raises_exception_when_parameter_not_set() -> # Act with pytest.raises(ParameterNotFoundError) as exinfo: - expression_runtime.evaluate(expression, state) + expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert str(exinfo.value) == "Parameter: 'variable' of type 'RunParameterType.LibraryVariables' not found" @@ -863,7 +864,7 @@ def test_json_nested_object_with_list_and_attributes(json_expression: str, acces expression_runtime = ExpressionRuntime() state = PipelineRunState() - actual = expression_runtime.evaluate(expression, state) + actual = expression_runtime.evaluate(expression, state, [], MockContext()) assert actual == expected @@ -966,11 +967,11 @@ def test_boolean_operators_short_circuit( # Act / Assert if isinstance(expected, bool): - actual = expression_runtime.evaluate(expression, state) + actual = expression_runtime.evaluate(expression, state, [], MockContext()) assert actual == expected else: with pytest.raises(expected): - expression_runtime.evaluate(expression, state) + expression_runtime.evaluate(expression, state, [], MockContext()) @pytest.mark.parametrize( @@ -1010,13 +1011,13 @@ def test_conditional_expression_with_branching( # Act / Assert if isinstance(expected, (str, int, bool, float)): - actual = expression_runtime.evaluate(expression, state) + actual = expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert actual == expected else: with pytest.raises(expected): - expression_runtime.evaluate(expression, state) + expression_runtime.evaluate(expression, state, [], MockContext()) @pytest.mark.parametrize( @@ -1038,7 +1039,7 @@ def test_complex_expression_with_missing_parameter(run_parameter_type: RunParame # Act with pytest.raises(ParameterNotFoundError) as exinfo: - expression_runtime.evaluate(expression, state) + expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert str(exinfo.value) == f"Parameter: 'parameter2' of type '{run_parameter_type}' not found" @@ -1064,7 +1065,7 @@ def test_complex_expression_with_missing_parameter_with_same_name_of_another_typ # Act with pytest.raises(ParameterNotFoundError) as exinfo: - expression_runtime.evaluate(expression, state) + expression_runtime.evaluate(expression, state, [], MockContext()) # Assert assert str(exinfo.value) == f"Parameter: 'parameter' of type '{run_parameter_type}' not found" diff --git a/tests/unit/models/activities/base/test_activity.py b/tests/unit/models/activities/base/test_activity.py index 9eb080f6..047e019a 100644 --- a/tests/unit/models/activities/base/test_activity.py +++ b/tests/unit/models/activities/base/test_activity.py @@ -1,6 +1,7 @@ import pytest from data_factory_testing_framework import TestFramework, TestFrameworkType from data_factory_testing_framework.models import DataFactoryElement +from data_factory_testing_framework.models._pipeline import Pipeline from data_factory_testing_framework.models.activities import Activity, ExecutePipelineActivity from data_factory_testing_framework.state import DependencyCondition, PipelineRunState, RunParameter, RunParameterType @@ -70,11 +71,13 @@ def test_dependency_condition_completed_is_false_when_no_activity_result_is_set( assert result is False -def test_evaluate_when_no_status_is_set_should_set_status_to_succeeded() -> None: +def test_evaluate_when_no_status_is_set_should_set_status_to_succeeded(pipeline: Pipeline) -> None: # Arrange pipeline_activity = Activity(name="activity", type="WebActivity", dependsOn=[]) state = PipelineRunState() + pipeline_activity._pipeline = pipeline + # Act pipeline_activity.evaluate(state) @@ -82,7 +85,7 @@ def test_evaluate_when_no_status_is_set_should_set_status_to_succeeded() -> None assert pipeline_activity.status == DependencyCondition.Succeeded -def test_evaluate_is_evaluating_expressions_inside_dict() -> None: +def test_evaluate_is_evaluating_expressions_inside_dict(pipeline: Pipeline) -> None: # Arrange pipeline_activity = ExecutePipelineActivity( name="activity", @@ -94,6 +97,9 @@ def test_evaluate_is_evaluating_expressions_inside_dict() -> None: }, depends_on=[], ) + + pipeline_activity._pipeline = pipeline + state = PipelineRunState( parameters=[ RunParameter(RunParameterType.Pipeline, "url", "example.com"), diff --git a/tests/unit/models/activities/conftest.py b/tests/unit/models/activities/conftest.py new file mode 100644 index 00000000..3ade16b3 --- /dev/null +++ b/tests/unit/models/activities/conftest.py @@ -0,0 +1,16 @@ +import pytest +from data_factory_testing_framework.models._pipeline import Pipeline +from data_factory_testing_framework.state import RunParameter, RunParameterType + + +@pytest.fixture +def pipeline() -> Pipeline: + """Fixture to create a sample pipeline.""" + return Pipeline( + pipeline_id="sample_pipeline", + name="SamplePipeline", + activities=[], + parameters=[ + RunParameter(name="param1", value="default_value", parameter_type=RunParameterType.Pipeline), + ], + ) \ No newline at end of file diff --git a/tests/unit/models/activities/control_activities/test_filter_activity.py b/tests/unit/models/activities/control_activities/test_filter_activity.py index 84513d1d..8a6de5c6 100644 --- a/tests/unit/models/activities/control_activities/test_filter_activity.py +++ b/tests/unit/models/activities/control_activities/test_filter_activity.py @@ -3,8 +3,9 @@ from data_factory_testing_framework.exceptions._control_activity_expression_evaluated_not_to_expected_type import ( ControlActivityExpressionEvaluatedNotToExpectedTypeError, ) -from data_factory_testing_framework.models import DataFactoryElement, DataFactoryObjectType, Pipeline +from data_factory_testing_framework.models import DataFactoryElement, DataFactoryObjectType from data_factory_testing_framework.models.activities import FilterActivity +from data_factory_testing_framework.models._pipeline import Pipeline from data_factory_testing_framework.state import PipelineRunState, RunParameter, RunParameterType @@ -19,7 +20,7 @@ ([-1, 3, 4], [-1, 3]), ], ) -def test_filter_activity_on_range_of_values(input_values: [], expected_filtered_values: []) -> None: +def test_filter_activity_on_range_of_values(input_values: list, expected_filtered_values: list) -> None: # Arrange test_framework = TestFramework(framework_type=TestFrameworkType.Fabric) pipeline = Pipeline( @@ -42,7 +43,6 @@ def test_filter_activity_on_range_of_values(input_values: [], expected_filtered_ ), ], ) - # Act activities = test_framework.evaluate_pipeline( pipeline, @@ -60,7 +60,8 @@ def test_filter_activity_on_range_of_values(input_values: [], expected_filtered_ @pytest.mark.parametrize(("evaluated_value"), [1, 1.1, "string-value", {}, True, None]) def test_filter_activity_evaluated_raises_error_when_evaluated_value_is_not_a_list( - evaluated_value: DataFactoryObjectType + evaluated_value: DataFactoryObjectType, + pipeline: Pipeline, ) -> None: # Arrange state = PipelineRunState(parameters=[RunParameter(RunParameterType.Pipeline, "input_values", evaluated_value)]) @@ -72,6 +73,8 @@ def test_filter_activity_evaluated_raises_error_when_evaluated_value_is_not_a_li }, ) + filter_activity._pipeline = pipeline + # Act with pytest.raises(ControlActivityExpressionEvaluatedNotToExpectedTypeError) as ex_info: filter_activity.evaluate(state) diff --git a/tests/unit/models/activities/control_activities/test_for_each_activity.py b/tests/unit/models/activities/control_activities/test_for_each_activity.py index 98120832..95ba8342 100644 --- a/tests/unit/models/activities/control_activities/test_for_each_activity.py +++ b/tests/unit/models/activities/control_activities/test_for_each_activity.py @@ -5,29 +5,37 @@ ) from data_factory_testing_framework.models import DataFactoryElement, DataFactoryObjectType from data_factory_testing_framework.models.activities import ForEachActivity, SetVariableActivity +from data_factory_testing_framework.models._pipeline import Pipeline from data_factory_testing_framework.state import PipelineRunState, PipelineRunVariable, RunParameter, RunParameterType -def test_when_evaluate_child_activities_then_should_return_the_activity_with_item_expression_evaluated() -> None: +def test_when_evaluate_child_activities_then_should_return_the_activity_with_item_expression_evaluated(pipeline: Pipeline) -> None: # Arrange test_framework = TestFramework(TestFrameworkType.Fabric) - for_each_activity = ForEachActivity( - name="ForEachActivity", - typeProperties={ - "items": DataFactoryElement("@split('a,b,c', ',')"), - }, - activities=[ - SetVariableActivity( + + set_variable_activity = SetVariableActivity( name="setVariable", typeProperties={ "variableName": "variable", "value": DataFactoryElement("@item()"), }, depends_on=[], - ), + ) + + for_each_activity = ForEachActivity( + name="ForEachActivity", + typeProperties={ + "items": DataFactoryElement("@split('a,b,c', ',')"), + }, + activities=[ + set_variable_activity ], depends_on=[], ) + + for_each_activity._pipeline = pipeline + set_variable_activity._pipeline = pipeline + state = PipelineRunState( variables=[ PipelineRunVariable(name="variable", default_value=""), @@ -59,7 +67,7 @@ def test_when_evaluate_child_activities_then_should_return_the_activity_with_ite @pytest.mark.parametrize(("evaluated_value"), [1, 1.1, "string-value", {}, True, None]) -def test_evaluated_raises_error_when_evaluated_value_is_not_a_list(evaluated_value: DataFactoryObjectType) -> None: +def test_evaluated_raises_error_when_evaluated_value_is_not_a_list(evaluated_value: DataFactoryObjectType, pipeline: Pipeline) -> None: # Arrange state = PipelineRunState(parameters=[RunParameter(RunParameterType.Pipeline, "input_values", evaluated_value)]) foreach_activity = ForEachActivity( @@ -71,6 +79,8 @@ def test_evaluated_raises_error_when_evaluated_value_is_not_a_list(evaluated_val depends_on=[], ) + foreach_activity._pipeline = pipeline + # Act with pytest.raises(ControlActivityExpressionEvaluatedNotToExpectedTypeError) as ex_info: foreach_activity.evaluate(state) diff --git a/tests/unit/models/activities/control_activities/test_if_condition_activity.py b/tests/unit/models/activities/control_activities/test_if_condition_activity.py index 2a50fa62..e79770e0 100644 --- a/tests/unit/models/activities/control_activities/test_if_condition_activity.py +++ b/tests/unit/models/activities/control_activities/test_if_condition_activity.py @@ -8,9 +8,10 @@ from data_factory_testing_framework.models import DataFactoryElement, DataFactoryObjectType from data_factory_testing_framework.models.activities import IfConditionActivity, SetVariableActivity from data_factory_testing_framework.state import PipelineRunState, PipelineRunVariable, RunParameter, RunParameterType +from data_factory_testing_framework.models._pipeline import Pipeline -def test_when_evaluated_should_evaluate_expression() -> None: +def test_when_evaluated_should_evaluate_expression(pipeline: Pipeline) -> None: # Arrange activity = IfConditionActivity( name="IfConditionActivity", @@ -18,6 +19,7 @@ def test_when_evaluated_should_evaluate_expression() -> None: if_false_activities=[], typeProperties={"expression": DataFactoryElement("@equals(1, 1)")}, ) + activity._pipeline = pipeline # Act activity.evaluate(PipelineRunState()) @@ -33,6 +35,7 @@ def test_when_evaluated_should_evaluate_expression() -> None: def test_when_evaluated_should_evaluate_correct_child_activities( expression_outcome: bool, expected_activity_name: str, + pipeline: Pipeline, ) -> None: # Arrange test_framework = TestFramework(framework_type=TestFrameworkType.Fabric) @@ -62,6 +65,10 @@ def test_when_evaluated_should_evaluate_correct_child_activities( ], ) + activity._pipeline = pipeline + activity.if_true_activities[0]._pipeline = pipeline + activity.if_false_activities[0]._pipeline = pipeline + state = PipelineRunState( variables=[ PipelineRunVariable(name="variable", default_value=""), @@ -97,7 +104,7 @@ def test_evaluate_pipeline_should_pass_iteration_item_to_child_activities() -> N @pytest.mark.parametrize(("evaluated_value"), [1, 1.1, "string-value", {}, [], None]) -def test_evaluated_raises_error_when_evaluated_value_is_not_a_bool(evaluated_value: DataFactoryObjectType) -> None: +def test_evaluated_raises_error_when_evaluated_value_is_not_a_bool(evaluated_value: DataFactoryObjectType, pipeline: Pipeline) -> None: # Arrange state = PipelineRunState(parameters=[RunParameter(RunParameterType.Pipeline, "input_values", evaluated_value)]) activity = IfConditionActivity( @@ -109,6 +116,8 @@ def test_evaluated_raises_error_when_evaluated_value_is_not_a_bool(evaluated_val }, ) + activity._pipeline = pipeline + # Act with pytest.raises(ControlActivityExpressionEvaluatedNotToExpectedTypeError) as ex_info: activity.evaluate(state) diff --git a/tests/unit/models/activities/control_activities/test_switch_activity.py b/tests/unit/models/activities/control_activities/test_switch_activity.py index f840c884..00d9d2f2 100644 --- a/tests/unit/models/activities/control_activities/test_switch_activity.py +++ b/tests/unit/models/activities/control_activities/test_switch_activity.py @@ -10,7 +10,7 @@ from data_factory_testing_framework.state import PipelineRunState, PipelineRunVariable, RunParameter, RunParameterType -def test_when_evaluated_should_evaluate_expression() -> None: +def test_when_evaluated_should_evaluate_expression(pipeline: Pipeline) -> None: # Arrange activity = SwitchActivity( name="SwitchActivity", @@ -18,6 +18,7 @@ def test_when_evaluated_should_evaluate_expression() -> None: cases_activities={}, typeProperties={"on": DataFactoryElement("@concat('case_', '1')")}, ) + activity._pipeline = pipeline # Act activity.evaluate(PipelineRunState()) @@ -105,14 +106,14 @@ def test_evaluate_pipeline_should_pass_iteration_item_to_child_activities() -> N evaluator = Mock(return_value=[]) # Act - list(activity.evaluate_control_activities(state, evaluator)) + list(activity.evaluate_control_activities(state, evaluator, [])) # Assert assert evaluator.call_args[0][1].iteration_item == "some-item" @pytest.mark.parametrize(("evaluated_value"), [1, 1.1, True, {}, [], None]) -def test_evaluated_raises_error_when_evaluated_value_is_not_a_str(evaluated_value: DataFactoryObjectType) -> None: +def test_evaluated_raises_error_when_evaluated_value_is_not_a_str(evaluated_value: DataFactoryObjectType, pipeline: Pipeline) -> None: # Arrange state = PipelineRunState(parameters=[RunParameter(RunParameterType.Pipeline, "input_values", evaluated_value)]) activity = SwitchActivity( @@ -122,6 +123,8 @@ def test_evaluated_raises_error_when_evaluated_value_is_not_a_str(evaluated_valu typeProperties={"on": DataFactoryElement("@pipeline().parameters.input_values")}, ) + activity._pipeline = pipeline + # Act with pytest.raises(ControlActivityExpressionEvaluatedNotToExpectedTypeError) as ex_info: activity.evaluate(state) diff --git a/tests/unit/models/activities/control_activities/test_until_activity.py b/tests/unit/models/activities/control_activities/test_until_activity.py index 6e81b0b4..5b8a68a1 100644 --- a/tests/unit/models/activities/control_activities/test_until_activity.py +++ b/tests/unit/models/activities/control_activities/test_until_activity.py @@ -1,3 +1,4 @@ +from typing import List from unittest.mock import Mock import pytest @@ -5,12 +6,14 @@ from data_factory_testing_framework.exceptions._control_activity_expression_evaluated_not_to_expected_type import ( ControlActivityExpressionEvaluatedNotToExpectedTypeError, ) +from data_factory_testing_framework.mock_context import MockContext from data_factory_testing_framework.models import DataFactoryElement, DataFactoryObjectType from data_factory_testing_framework.models.activities import SetVariableActivity, UntilActivity from data_factory_testing_framework.state import PipelineRunState, PipelineRunVariable, RunParameter, RunParameterType +from data_factory_testing_framework.models._pipeline import Pipeline -def test_when_evaluate_until_activity_should_repeat_until_expression_is_true(monkeypatch: pytest.MonkeyPatch) -> None: +def test_when_evaluate_until_activity_should_repeat_until_expression_is_true(monkeypatch: pytest.MonkeyPatch, pipeline: Pipeline) -> None: # Arrange test_framework = TestFramework(framework_type=TestFrameworkType.Fabric) until_activity = UntilActivity( @@ -30,6 +33,8 @@ def test_when_evaluate_until_activity_should_repeat_until_expression_is_true(mon ], depends_on=[], ) + until_activity._pipeline = pipeline + until_activity.activities[0]._pipeline = pipeline state = PipelineRunState( variables=[ @@ -38,7 +43,7 @@ def test_when_evaluate_until_activity_should_repeat_until_expression_is_true(mon ) # Act - monkeypatch.setattr(until_activity.expression, "evaluate", lambda state: False) + monkeypatch.setattr(until_activity.expression, "evaluate", lambda state, mocks, mock_context: False) activities = test_framework.evaluate_activity(until_activity, state) # Assert @@ -50,7 +55,7 @@ def test_when_evaluate_until_activity_should_repeat_until_expression_is_true(mon assert set_variable_activity is not None assert set_variable_activity.name == "setVariable" - monkeypatch.setattr(until_activity.expression, "evaluate", lambda state: True) + monkeypatch.setattr(until_activity.expression, "evaluate", lambda state, mocks, mock_context: True) # Assert that there are no more activities with pytest.raises(StopIteration): diff --git a/tests/unit/models/activities/test_append_variable_activity.py b/tests/unit/models/activities/test_append_variable_activity.py index 29b1b5af..0ca185f3 100644 --- a/tests/unit/models/activities/test_append_variable_activity.py +++ b/tests/unit/models/activities/test_append_variable_activity.py @@ -7,6 +7,7 @@ ) from data_factory_testing_framework.models import DataFactoryElement from data_factory_testing_framework.models.activities import AppendVariableActivity +from data_factory_testing_framework.models._pipeline import Pipeline from data_factory_testing_framework.state import PipelineRunState, PipelineRunVariable @@ -19,7 +20,7 @@ ], ) def test_when_int_variable_appended_then_state_variable_should_be_set( - initial_value: List[int], appended_value: int, expected_value: List[int] + initial_value: List[int], appended_value: int, expected_value: List[int], pipeline: Pipeline ) -> None: # Arrange TestFramework(framework_type=TestFrameworkType.Fabric) @@ -31,6 +32,7 @@ def test_when_int_variable_appended_then_state_variable_should_be_set( "value": DataFactoryElement(str(appended_value)), }, ) + set_variable_activity._pipeline = pipeline state = PipelineRunState( variables=[ PipelineRunVariable(name=variable_name, default_value=initial_value), @@ -45,7 +47,7 @@ def test_when_int_variable_appended_then_state_variable_should_be_set( assert variable.value == expected_value -def test_when_unknown_variable_evaluated_then_should_raise_exception() -> None: +def test_when_unknown_variable_evaluated_then_should_raise_exception(pipeline: Pipeline) -> None: # Arrange TestFramework(framework_type=TestFrameworkType.Fabric) variable_name = "TestVariable" @@ -56,6 +58,7 @@ def test_when_unknown_variable_evaluated_then_should_raise_exception() -> None: "value": DataFactoryElement("TestValue"), }, ) + set_variable_activity._pipeline = pipeline state = PipelineRunState() # Act diff --git a/tests/unit/models/activities/test_execute_pipeline_activity_parameters.py b/tests/unit/models/activities/test_execute_pipeline_activity_parameters.py index fa94ea96..78c39388 100644 --- a/tests/unit/models/activities/test_execute_pipeline_activity_parameters.py +++ b/tests/unit/models/activities/test_execute_pipeline_activity_parameters.py @@ -1,9 +1,12 @@ from data_factory_testing_framework.models import DataFactoryElement +from data_factory_testing_framework.models._pipeline import Pipeline from data_factory_testing_framework.models.activities import ExecutePipelineActivity from data_factory_testing_framework.state import PipelineRunState, RunParameter, RunParameterType -def test_execute_pipeline_activity_evaluates_parameters() -> None: +def test_execute_pipeline_activity_evaluates_parameters( + pipeline: Pipeline, +) -> None: # Arrange execute_pipeline_activity = ExecutePipelineActivity( name="ExecutePipelineActivity", @@ -14,6 +17,8 @@ def test_execute_pipeline_activity_evaluates_parameters() -> None: }, depends_on=[], ) + execute_pipeline_activity._pipeline = pipeline + state = PipelineRunState( parameters=[ RunParameter(name="param1", value="value1", parameter_type=RunParameterType.Pipeline), @@ -29,13 +34,17 @@ def test_execute_pipeline_activity_evaluates_parameters() -> None: assert activity.parameters["url"].result == "value1" -def test_execute_pipeline_activity_evaluates_no_parameters() -> None: +def test_execute_pipeline_activity_evaluates_no_parameters( + pipeline: Pipeline, +) -> None: # Arrange execute_pipeline_activity = ExecutePipelineActivity( name="ExecutePipelineActivity", typeProperties={}, depends_on=[], ) + execute_pipeline_activity._pipeline = pipeline + state = PipelineRunState() # Act diff --git a/tests/unit/models/activities/test_fail_activity.py b/tests/unit/models/activities/test_fail_activity.py index 9ee79f9a..22dc43e0 100644 --- a/tests/unit/models/activities/test_fail_activity.py +++ b/tests/unit/models/activities/test_fail_activity.py @@ -1,9 +1,10 @@ from data_factory_testing_framework.models import DataFactoryElement from data_factory_testing_framework.models.activities import FailActivity from data_factory_testing_framework.state import DependencyCondition, PipelineRunState +from data_factory_testing_framework.models._pipeline import Pipeline -def test_fail_activity_evaluates_to_failed_result() -> None: +def test_fail_activity_evaluates_to_failed_result(pipeline: Pipeline) -> None: # Arrange fail_activity = FailActivity( name="FailActivity", @@ -13,7 +14,7 @@ def test_fail_activity_evaluates_to_failed_result() -> None: }, depends_on=[], ) - + fail_activity._pipeline = pipeline state = PipelineRunState() # Act diff --git a/tests/unit/models/activities/test_set_variable_activity.py b/tests/unit/models/activities/test_set_variable_activity.py index 3e4e5c58..432319a9 100644 --- a/tests/unit/models/activities/test_set_variable_activity.py +++ b/tests/unit/models/activities/test_set_variable_activity.py @@ -5,10 +5,11 @@ ) from data_factory_testing_framework.models import DataFactoryElement from data_factory_testing_framework.models.activities import SetVariableActivity +from data_factory_testing_framework.models._pipeline import Pipeline from data_factory_testing_framework.state import PipelineRunState, PipelineRunVariable -def test_when_string_variable_evaluated_then_state_variable_should_be_set() -> None: +def test_when_string_variable_evaluated_then_state_variable_should_be_set(pipeline: Pipeline) -> None: # Arrange TestFramework(framework_type=TestFrameworkType.Fabric) variable_name = "TestVariable" @@ -19,6 +20,7 @@ def test_when_string_variable_evaluated_then_state_variable_should_be_set() -> N "value": DataFactoryElement("TestValue"), }, ) + set_variable_activity._pipeline = pipeline state = PipelineRunState( variables=[ PipelineRunVariable(name=variable_name, default_value=""), @@ -33,7 +35,7 @@ def test_when_string_variable_evaluated_then_state_variable_should_be_set() -> N assert variable.value == "TestValue" -def test_when_unknown_variable_evaluated_then_should_raise_exception() -> None: +def test_when_unknown_variable_evaluated_then_should_raise_exception(pipeline: Pipeline) -> None: # Arrange TestFramework(framework_type=TestFrameworkType.Fabric) variable_name = "TestVariable" @@ -44,6 +46,7 @@ def test_when_unknown_variable_evaluated_then_should_raise_exception() -> None: "value": DataFactoryElement("TestValue"), }, ) + set_variable_activity._pipeline = pipeline state = PipelineRunState() # Act