specmatic · burdeamit · Apr 30, 2024 · Apr 30, 2024
diff --git a/specmatic/generators/contract_test.py b/specmatic/generators/contract_test.py
@@ -0,0 +1,5 @@
+class ContractTest:
+    def __init__(self, name: str, passed: bool, error_message: str = ""):
+        self.name = name
+        self.passed = passed
+        self.error_message = error_message
diff --git a/specmatic/generators/pytest_generator.py b/specmatic/generators/pytest_generator.py
@@ -10,8 +10,13 @@ def __init__(self, test_class, junit_report_path):
         self.junit_report_path = junit_report_path
 
     def generate(self):
-        self.generate_tests(self.junit_report_path, self.test_class,  PyTestGenerator._generate_passing_test,
-                            PyTestGenerator._generate_failing_test)
+        contract_tests = self.extract_contract_tests(self.junit_report_path)
+        for contract_test in contract_tests:
+            if contract_test.passed:
+                setattr(self.test_class, contract_test.name, PyTestGenerator._generate_passing_test())
+            else:
+                setattr(self.test_class, contract_test.name, PyTestGenerator._generate_failing_test(contract_test.error_message))
+
 
     @staticmethod
     def _generate_passing_test():

diff --git a/specmatic/generators/test_generator_base.py b/specmatic/generators/test_generator_base.py
@@ -1,15 +1,28 @@
 import xml.etree.ElementTree as ET
+from typing import List
+
+from specmatic.generators.contract_test import ContractTest
 
 
 class TestGeneratorBase:
     @staticmethod
-    def generate_tests(junit_report_path, test_class, passing_test_fn, failing_test_fn):
+    def extract_contract_tests(junit_report_path) -> List[ContractTest]:
+        def extract_test_name(testcase) -> str:
+            return "test_" + testcase.get('name').strip()
+
+        def get_error_message(failure) -> str:
+            return "" if failure is None else failure.get('message')
+
         root = ET.parse(junit_report_path).getroot()
-        for testcase in root.iter('testcase'):
-            scenario = testcase.find('system-out').text.split('display-name:')[1].strip()
-            test_name = "test_" + scenario
-            failure = testcase.find('failure')
-            if failure is None:
-                setattr(test_class, test_name, passing_test_fn())
-            else:
-                setattr(test_class, test_name, failing_test_fn(failure.get('message') + failure.text))
+
+        contract_tests = [
+            ContractTest(
+                name=extract_test_name(testcase),
+                passed=failure is None,
+                error_message=get_error_message(failure)
+            )
+            for testcase in root.iter('testcase')
+            for failure in [testcase.find('failure')]
+        ]
+        print("Extracted " + str(len(contract_tests)) + " contract test(s)")
+        return contract_tests
diff --git a/specmatic/generators/unittest_generator.py b/specmatic/generators/unittest_generator.py
@@ -22,5 +22,10 @@ def test(self):
         return test
 
     def generate(self):
-        self.generate_tests(self.junit_report_path, self.test_class, UnitTestGenerator._gen_passing_test,
-                            UnitTestGenerator._gen_failing_test)
+        contract_tests = self.extract_contract_tests(self.junit_report_path)
+        for contract_test in contract_tests:
+            if contract_test.passed:
+                setattr(self.test_class, contract_test.name, UnitTestGenerator._gen_passing_test())
+            else:
+                setattr(self.test_class, contract_test.name,
+                        UnitTestGenerator._gen_failing_test(contract_test.error_message))
diff --git a/test/asgi/test_contract_with_fastapi_app_with_app_server_started_externally.py b/test/asgi/test_contract_with_fastapi_app_with_app_server_started_externally.py
@@ -13,7 +13,7 @@
 stub_host = "127.0.0.1"
 stub_port = 8080
 
-expectation_json_file = PROJECT_ROOT + '/test/data/expectation.json'
+expectation_json_file = PROJECT_ROOT + '/test/resources/data/expectation.json'
 app_module = PROJECT_ROOT + '/test/sanic_app'
 
 app_server = ASGIAppServer('test.apps.fast_api:app', app_host, app_port)

diff --git a/test/asgi/test_contract_with_fastapi_app_with_coverage_by_setting_endpoints_api_url.py b/test/asgi/test_contract_with_fastapi_app_with_coverage_by_setting_endpoints_api_url.py
@@ -13,7 +13,7 @@
 stub_host = "127.0.0.1"
 stub_port = 8080
 
-expectation_json_file = PROJECT_ROOT + '/test/data/expectation.json'
+expectation_json_file = PROJECT_ROOT + '/test/resources/data/expectation.json'
 app_module = PROJECT_ROOT + '/test/sanic_app'
 
 app_server = ASGIAppServer('test.apps.fast_api:app', app_host, app_port)

diff --git a/test/asgi/test_contract_with_sanic_app_with_autowiring.py b/test/asgi/test_contract_with_sanic_app_with_autowiring.py
@@ -12,7 +12,7 @@
 stub_host = "127.0.0.1"
 stub_port = 8080
 
-expectation_json_file = PROJECT_ROOT + '/test/data/expectation.json'
+expectation_json_file = PROJECT_ROOT + '/test/resources/data/expectation.json'
 app_module = PROJECT_ROOT + '/test/sanic_app'
 
 config_ini_path = get_project_root() + '/test/config.ini'

diff --git a/test/asgi/test_contract_with_sanic_app_with_coverage.py b/test/asgi/test_contract_with_sanic_app_with_coverage.py
@@ -11,7 +11,7 @@
 stub_host = "127.0.0.1"
 stub_port = 8080
 
-expectation_json_file = PROJECT_ROOT + '/test/data/expectation.json'
+expectation_json_file = PROJECT_ROOT + '/test/resources/data/expectation.json'
 
 
 class TestContract:

diff --git a/test/asgi/test_contract_with_sanic_app_with_coverage_where_adapter_is_explicitly_set.py b/test/asgi/test_contract_with_sanic_app_with_coverage_where_adapter_is_explicitly_set.py
@@ -14,7 +14,7 @@
 stub_host = "127.0.0.1"
 stub_port = 8080
 
-expectation_json_file = PROJECT_ROOT + '/test/data/expectation.json'
+expectation_json_file = PROJECT_ROOT + '/test/resources/data/expectation.json'
 app_module = PROJECT_ROOT + '/test/sanic_app'
 
 

diff --git a/test/negative/test_negative_scenarios.py b/test/negative/test_negative_scenarios.py
@@ -12,8 +12,8 @@
 stub_host = "127.0.0.1"
 stub_port = 8080
 
-expectation_json_file = PROJECT_ROOT + '/test/data/expectation.json'
-invalid_expectation_json_file = PROJECT_ROOT + '/test/data/invalid_expectation.json'
+expectation_json_file = PROJECT_ROOT + '/test/resources/data/expectation.json'
+invalid_expectation_json_file = PROJECT_ROOT + '/test/resources/data/invalid_expectation.json'
 app_module = PROJECT_ROOT + '/test/sanic_app'
 
 

diff --git a/test/resources/TEST-junit-jupiter.xml b/test/resources/TEST-junit-jupiter.xml
@@ -0,0 +1,121 @@
+<testsuite errors="0" failures="2" hostname="apples-MacBook-Pro.local" name="Contract Tests" skipped="0" tests="5"
+           time="2.234" timestamp="2024-04-22T23:27:12">
+    <properties>
+        <property name="apple.awt.application.name" value="SpecmaticApplication"/>
+        <property name="environment" value=""/>
+        <property name="file.encoding" value="UTF-8"/>
+        <property name="file.separator" value="/"/>
+        <property name="ftp.nonProxyHosts" value="local|*.local|169.254/16|*.169.254/16"/>
+        <property name="host" value="localhost"/>
+        <property name="http.nonProxyHosts" value="local|*.local|169.254/16|*.169.254/16"/>
+        <property name="java.awt.headless" value="true"/>
+        <property name="java.class.path" value="/Users/jaydeep/specmatic/specmatic.jar"/>
+        <property name="java.class.version" value="63.0"/>
+        <property name="java.home" value="/Library/Java/JavaVirtualMachines/temurin-19.jdk/Contents/Home"/>
+        <property name="java.io.tmpdir" value="/var/folders/t0/n9_yhvbn5yx6cs4hwk48s3gm0000gp/T/"/>
+        <property name="java.library.path"
+                  value="/Users/jaydeep/Library/Java/Extensions:/Library/Java/Extensions:/Network/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java:."/>
+        <property name="java.runtime.name" value="OpenJDK Runtime Environment"/>
+        <property name="java.runtime.version" value="19.0.2+7"/>
+        <property name="java.specification.name" value="Java Platform API Specification"/>
+        <property name="java.specification.vendor" value="Oracle Corporation"/>
+        <property name="java.specification.version" value="19"/>
+        <property name="java.vendor" value="Eclipse Adoptium"/>
+        <property name="java.vendor.url" value="https://adoptium.net/"/>
+        <property name="java.vendor.url.bug" value="https://github.com/adoptium/adoptium-support/issues"/>
+        <property name="java.vendor.version" value="Temurin-19.0.2+7"/>
+        <property name="java.version" value="19.0.2"/>
+        <property name="java.version.date" value="2023-01-17"/>
+        <property name="java.vm.compressedOopsMode" value="Zero based"/>
+        <property name="java.vm.info" value="mixed mode"/>
+        <property name="java.vm.name" value="OpenJDK 64-Bit Server VM"/>
+        <property name="java.vm.specification.name" value="Java Virtual Machine Specification"/>
+        <property name="java.vm.specification.vendor" value="Oracle Corporation"/>
+        <property name="java.vm.specification.version" value="19"/>
+        <property name="java.vm.vendor" value="Eclipse Adoptium"/>
+        <property name="java.vm.version" value="19.0.2+7"/>
+        <property name="jdk.debug" value="release"/>
+        <property name="line.separator" value=" "/>
+        <property name="native.encoding" value="UTF-8"/>
+        <property name="os.arch" value="aarch64"/>
+        <property name="os.name" value="Mac OS X"/>
+        <property name="os.version" value="13.0"/>
+        <property name="path.separator" value=":"/>
+        <property name="port" value="5000"/>
+        <property name="protocol" value="http"/>
+        <property name="socksNonProxyHosts" value="local|*.local|169.254/16|*.169.254/16"/>
+        <property name="stderr.encoding" value="UTF-8"/>
+        <property name="stdout.encoding" value="UTF-8"/>
+        <property name="suggestions" value=""/>
+        <property name="suggestionsPath" value=""/>
+        <property name="sun.arch.data.model" value="64"/>
+        <property name="sun.boot.library.path"
+                  value="/Library/Java/JavaVirtualMachines/temurin-19.jdk/Contents/Home/lib"/>
+        <property name="sun.cpu.endian" value="little"/>
+        <property name="sun.io.unicode.encoding" value="UnicodeBig"/>
+        <property name="sun.java.command"
+                  value="/Users/jaydeep/specmatic/specmatic.jar test --port=5000 --junitReportDir=."/>
+        <property name="sun.java.launcher" value="SUN_STANDARD"/>
+        <property name="sun.jnu.encoding" value="UTF-8"/>
+        <property name="sun.management.compiler" value="HotSpot 64-Bit Tiered Compilers"/>
+        <property name="timeout" value="60"/>
+        <property name="user.country" value="IN"/>
+        <property name="user.dir" value="/Users/jaydeep/znsio/specmatic-order-api-python"/>
+        <property name="user.home" value="/Users/jaydeep"/>
+        <property name="user.language" value="en"/>
+        <property name="user.name" value="jaydeep"/>
+        <property name="user.timezone" value="Asia/Kolkata"/>
+    </properties>
+    <testcase classname="in.specmatic.test.SpecmaticJUnitSupport"
+              name="+ve  Scenario: GET /products/(id:number) -&gt; 404 | EX:INVALID_ID" time="0.005">
+        <system-out><![CDATA[
+            unique-id:
+            [engine:junit-jupiter]/[class:in.specmatic.test.SpecmaticJUnitSupport]/[test-factory:contractTest()]/[dynamic-test:#6]
+            display-name: +ve Scenario: GET /products/(id:number) -> 404 | EX:INVALID_ID
+            ]]>
+        </system-out>
+    </testcase>
+    <testcase classname="in.specmatic.test.SpecmaticJUnitSupport"
+              name="+ve  Scenario: DELETE /products/(id:number) -&gt; 200 | EX:DELETE_PRODUCT" time="0.006">
+        <failure message="server error"></failure>
+        <system-out><![CDATA[
+            unique-id:
+            [engine:junit-jupiter]/[class:in.specmatic.test.SpecmaticJUnitSupport]/[test-factory:contractTest()]/[dynamic-test:#7]
+            display-name: +ve Scenario: DELETE /products/(id:number) -> 200 | EX:DELETE_PRODUCT
+            ]]>
+        </system-out>
+    </testcase>
+    <testcase classname="in.specmatic.test.SpecmaticJUnitSupport"
+              name="+ve  Scenario: POST /products/(id:number) -&gt; 200 | EX:UPDATE_DETAILS" time="0.005">
+        <system-out><![CDATA[
+            unique-id:
+            [engine:junit-jupiter]/[class:in.specmatic.test.SpecmaticJUnitSupport]/[test-factory:contractTest()]/[dynamic-test:#4]
+            display-name: +ve Scenario: POST /products/(id:number) -> 200 | EX:UPDATE_DETAILS
+            ]]>
+        </system-out>
+    </testcase>
+    <testcase classname="in.specmatic.test.SpecmaticJUnitSupport"
+              name="+ve  Scenario: GET /products/(id:number) -&gt; 200 | EX:GET_DETAILS" time="0.009">
+        <failure message="server error"></failure>
+        <system-out><![CDATA[
+            unique-id:
+            [engine:junit-jupiter]/[class:in.specmatic.test.SpecmaticJUnitSupport]/[test-factory:contractTest()]/[dynamic-test:#5]
+            display-name: +ve Scenario: GET /products/(id:number) -> 200 | EX:GET_DETAILS
+            ]]>
+        </system-out>
+    </testcase>
+    <testcase classname="in.specmatic.test.SpecmaticJUnitSupport"
+              name="+ve  Scenario: POST /products/(id:number) -&gt; 200 | EX:UPDATE_DETAILS" time="0.008">
+        <system-out><![CDATA[
+            unique-id:
+            [engine:junit-jupiter]/[class:in.specmatic.test.SpecmaticJUnitSupport]/[test-factory:contractTest()]/[dynamic-test:#2]
+            display-name: +ve Scenario: POST /products/(id:number) -> 200 | EX:UPDATE_DETAILS
+            ]]>
+        </system-out>
+    </testcase>
+    <system-out><![CDATA[
+        unique-id: [engine:junit-jupiter]
+        display-name: Contract Tests
+        ]]>
+    </system-out>
+</testsuite>
diff --git a/test/data/expectation.json → test/resources/data/expectation.json b/test/data/expectation.json → test/resources/data/expectation.json
diff --git a/test/data/invalid_expectation.json → test/resources/data/invalid_expectation.json b/test/data/invalid_expectation.json → test/resources/data/invalid_expectation.json
diff --git a/test/unit/test_test_generator_base.py b/test/unit/test_test_generator_base.py
@@ -0,0 +1,24 @@
+from specmatic.generators.test_generator_base import TestGeneratorBase
+from specmatic.utils import get_project_root
+
+
+class TestsForTestGeneratorBase:
+
+    @classmethod
+    def setup_class(cls):
+        junit_report_file = get_project_root() + '/test/resources/TEST-junit-jupiter.xml'
+        cls.contract_tests = TestGeneratorBase.extract_contract_tests(junit_report_file)
+
+    def test_extracts_all_testcases_from_junit_report(self):
+        assert len(self.contract_tests) == 5
+
+    def test_correctly_identifies_tests_as_passed_or_failed(self):
+        passed_tests = [contract_test for contract_test in self.contract_tests if contract_test.passed is True]
+        assert len(passed_tests) == 3
+
+        failed_tests = [contract_test for contract_test in self.contract_tests if contract_test.passed is False]
+        assert len(failed_tests) == 2
+
+    def test_extracts_failure_reason_for_failed_tests(self):
+        failed_tests = [contract_test for contract_test in self.contract_tests if contract_test.passed is False]
+        assert len([failed_test for failed_test in failed_tests if failed_test.error_message == 'server error']) == 2
diff --git a/test/wsgi/test_contract_using_api_with_autowiring.py b/test/wsgi/test_contract_using_api_with_autowiring.py
@@ -9,7 +9,7 @@
 stub_host = "127.0.0.1"
 stub_port = 8080
 PROJECT_ROOT = get_project_root()
-expectation_json_file = PROJECT_ROOT + '/test/data/expectation.json'
+expectation_json_file = PROJECT_ROOT + '/test/resources/data/expectation.json'
 
 
 class TestContract:

diff --git a/test/wsgi/test_contract_using_api_with_unittest.py b/test/wsgi/test_contract_using_api_with_unittest.py
@@ -7,7 +7,7 @@
 stub_host = "127.0.0.1"
 stub_port = 8080
 PROJECT_ROOT = get_project_root()
-expectation_json_file = PROJECT_ROOT + '/test/data/expectation.json'
+expectation_json_file = PROJECT_ROOT + '/test/resources/data/expectation.json'
 
 
 class TestContract(unittest.TestCase):

diff --git a/test/wsgi/test_contract_using_decorators_with_coverage.py b/test/wsgi/test_contract_using_decorators_with_coverage.py
@@ -11,7 +11,7 @@
 stub_port = 8080
 PROJECT_ROOT = get_project_root()
 APP_ROOT = PROJECT_ROOT + '/test'
-expectation_json_file = PROJECT_ROOT + '/test/data/expectation.json'
+expectation_json_file = PROJECT_ROOT + '/test/resources/data/expectation.json'
 
 
 @specmatic_contract_test(project_root=PROJECT_ROOT, appRouteAdapter=FlaskAppRouteAdapter(app))

diff --git a/test/wsgi/test_contract_with_api_coverage.py b/test/wsgi/test_contract_with_api_coverage.py
@@ -9,7 +9,7 @@
 stub_host = "127.0.0.1"
 stub_port = 8080
 PROJECT_ROOT = get_project_root()
-expectation_json_file = PROJECT_ROOT + '/test/data/expectation.json'
+expectation_json_file = PROJECT_ROOT + '/test/resources/data/expectation.json'
 
 
 class TestContract:

diff --git a/test/wsgi/test_contract_with_api_coverage_with_app_started_externally.py b/test/wsgi/test_contract_with_api_coverage_with_app_started_externally.py
@@ -10,7 +10,7 @@
 stub_host = "127.0.0.1"
 stub_port = 8080
 PROJECT_ROOT = get_project_root()
-expectation_json_file = PROJECT_ROOT + '/test/data/expectation.json'
+expectation_json_file = PROJECT_ROOT + '/test/resources/data/expectation.json'
 
 app_server = WSGIAppServer(app, app_host, app_port)
 app_server.start()

diff --git a/test/wsgi/test_contract_with_extra_arguments.py b/test/wsgi/test_contract_with_extra_arguments.py
@@ -8,7 +8,7 @@
 stub_host = "127.0.0.1"
 stub_port = 8080
 PROJECT_ROOT = get_project_root()
-expectation_json_file = PROJECT_ROOT + '/test/data/expectation.json'
+expectation_json_file = PROJECT_ROOT + '/test/resources/data/expectation.json'
 
 
 class TestContract:

diff --git a/test/wsgi/test_contract_with_generative_tests.py b/test/wsgi/test_contract_with_generative_tests.py
@@ -9,7 +9,7 @@
 stub_host = "127.0.0.1"
 stub_port = 8080
 PROJECT_ROOT = get_project_root()
-expectation_json_file = PROJECT_ROOT + '/test/data/expectation.json'
+expectation_json_file = PROJECT_ROOT + '/test/resources/data/expectation.json'
 
 
 class TestContract: