test: evaluate skipped tests, fix failing tests, add step in e2e workflow to mark build as failed (#351)

* update test_linode.py

* evaluate skipped tests, add steps in e2e workflow, fix flaky tests

* lint

* fix flaky tests in lke and volume

* add some safety wait until functions for reorder test

* update test_delete_interface_containing_vpc, test_linode_initate_migration

Author: ykim-akamai

.github/workflows/e2e-test-pr.yml

@@ -19,6 +19,8 @@ jobs:
     runs-on: ubuntu-latest
     if:
       github.event_name == 'workflow_dispatch' && inputs.sha != ''
+    env:
+      EXIT_STATUS: 0
 
     steps:
       - uses: actions-ecosystem/action-regex-match@v2
@@ -83,8 +85,8 @@ jobs:
           timestamp=$(date +'%Y%m%d%H%M')
           report_filename="${timestamp}_sdk_test_report.xml"
           status=0
-          if ! python3 -m pytest test/integration/${INTEGRATION_TEST_PATH} --junitxml="${report_filename}"; then
-            echo "Tests failed, but attempting to upload results anyway"
+          if ! python3 -m pytest test/integration/${INTEGRATION_TEST_PATH} --disable-warnings --junitxml="${report_filename}"; then
+            echo "EXIT_STATUS=1" >> $GITHUB_ENV
           fi
         env:
           LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }}
@@ -135,4 +137,13 @@ jobs:
               status: 'completed',
               conclusion: process.env.conclusion
             });
-            return result;
+            return result;
+
+      - name: Test Execution Status Handler
+        run: |
+          if [[ "$EXIT_STATUS" != 0 ]]; then
+            echo "Test execution contains failure(s)"
+            exit $EXIT_STATUS 
+          else
+            echo "Tests passed!"
+          fi

test/integration/conftest.py

@@ -155,11 +155,10 @@ def test_domain(test_linode_client):
 def test_volume(test_linode_client):
     client = test_linode_client
     timestamp = str(time.time_ns())
+    region = client.regions()[0]
     label = "TestSDK-" + timestamp
 
-    volume = client.volume_create(
-        label=label, region=get_region(client, {"Block Storage"})
-    )
+    volume = client.volume_create(label=label, region=region)
 
     yield volume
 

test/integration/helpers.py

@@ -111,7 +111,7 @@ def send_request_when_resource_available(
                 if time.time() - start_time > timeout:
                     raise TimeoutError(
                         "Timeout Error: resource is not available in"
-                        + timeout
+                        + str(timeout)
                         + "seconds"
                     )
                 time.sleep(10)

test/integration/models/test_linode.py

@@ -2,6 +2,7 @@
 from test.integration.helpers import (
     get_test_label,
     retry_sending_request,
+    send_request_when_resource_available,
     wait_for_condition,
 )
 
@@ -65,6 +66,53 @@ def linode_with_volume_firewall(test_linode_client):
     linode_instance.delete()
 
 
+@pytest.fixture(scope="session")
+def linode_for_network_interface_tests(test_linode_client):
+    client = test_linode_client
+    available_regions = client.regions()
+    chosen_region = available_regions[0]
+    timestamp = str(time.time_ns())
+    label = "TestSDK-" + timestamp
+
+    linode_instance, password = client.linode.instance_create(
+        "g6-nanode-1", chosen_region, image="linode/debian10", label=label
+    )
+
+    yield linode_instance
+
+    linode_instance.delete()
+
+
+@pytest.fixture(scope="session", autouse=True)
+def linode_for_disk_tests(test_linode_client):
+    client = test_linode_client
+    available_regions = client.regions()
+    chosen_region = available_regions[0]
+    label = get_test_label()
+
+    linode_instance, password = client.linode.instance_create(
+        "g6-nanode-1",
+        chosen_region,
+        image="linode/debian10",
+        label=label + "_long_tests",
+    )
+
+    time.sleep(10)
+
+    # Provisioning time
+    wait_for_condition(10, 300, get_status, linode_instance, "running")
+
+    time.sleep(10)
+
+    linode_instance.shutdown()
+
+    wait_for_condition(10, 100, get_status, linode_instance, "offline")
+
+    yield linode_instance
+
+    linode_instance.delete()
+
+
 @pytest.mark.smoke
 @pytest.fixture
 def create_linode_for_long_running_tests(test_linode_client):
@@ -289,22 +337,53 @@ def test_linode_volumes(linode_with_volume_firewall):
     assert "TestSDK" in volumes[0].label
 
 
-def test_linode_disk_duplicate(test_linode_client, create_linode):
-    pytest.skip("Need to find out the space sizing when duplicating disks")
-    linode = create_linode
+def wait_for_disk_status(disk: Disk, timeout):
+    start_time = time.time()
+    while True:
+        try:
+            if disk.status == "ready":
+                return disk.status
+        except ApiError:
+            if time.time() - start_time > timeout:
+                raise TimeoutError("Wait for condition timeout error")
+
+
+@pytest.mark.dependency()
+def test_disk_resize_and_duplicate(test_linode_client, linode_for_disk_tests):
+    linode = linode_for_disk_tests
+
+    disk = linode.disks[0]
+
+    disk.resize(5000)
+
+    # Using hard sleep instead of wait as the status shows ready when it is resizing
+    time.sleep(120)
 
     disk = test_linode_client.load(Disk, linode.disks[0].id, linode.id)
 
-    try:
-        dup_disk = disk.duplicate()
-        assert dup_disk.linode_id == linode.id
-    except ApiError as e:
-        assert e.status == 400
-        assert "Insufficient space" in str(e.json)
+    assert disk.size == 5000
+
+    dup_disk = disk.duplicate()
+
+    time.sleep(40)
+
+    wait_for_disk_status(dup_disk, 120)
+
+    assert dup_disk.linode_id == linode.id
+
+
+@pytest.mark.dependency(depends=["test_disk_resize_and_duplicate"])
+def test_linode_create_disk(test_linode_client, linode_for_disk_tests):
+    linode = test_linode_client.load(Instance, linode_for_disk_tests.id)
+
+    disk = linode.disk_create(size=500)
+
+    wait_for_disk_status(disk, 120)
+
+    assert disk.linode_id == linode.id
 
 
 def test_linode_instance_password(create_linode_for_pass_reset):
-    pytest.skip("Failing due to mismatched request body")
     linode = create_linode_for_pass_reset[0]
     password = create_linode_for_pass_reset[1]
 
@@ -343,25 +422,14 @@ def test_linode_initate_migration(test_linode_client):
 
     wait_for_condition(10, 100, get_status, linode, "running")
     # Says it could take up to ~6 hrs for migration to fully complete
-    linode.initiate_migration(region="us-central")
 
-    res = linode.delete()
-
-    assert res
-
-
-def test_linode_create_disk(create_linode):
-    pytest.skip(
-        "Pre-requisite for the test account need to comply with this test"
+    send_request_when_resource_available(
+        300, linode.initiate_migration, "us-central"
     )
-    linode = create_linode
-    disk, gen_pass = linode.disk_create()
 
+    res = linode.delete()
 
-def test_disk_resize():
-    pytest.skip(
-        "Pre-requisite for the test account need to comply with this test"
-    )
+    assert res
 
 
 def test_config_update_interfaces(create_linode):
@@ -387,19 +455,9 @@ def test_config_update_interfaces(create_linode):
 
 
 def test_get_config(test_linode_client, create_linode):
-    pytest.skip(
-        "Model get method: client.load(Config, 123, 123) does not work..."
-    )
     linode = create_linode
-    json = test_linode_client.get(
-        "linode/instances/"
-        + str(linode.id)
-        + "/configs/"
-        + str(linode.configs[0].id)
-    )
-    config = Config(
-        test_linode_client, linode.id, linode.configs[0].id, json=json
-    )
+
+    config = test_linode_client.load(Config, linode.configs[0].id, linode.id)
 
     assert config.id == linode.configs[0].id
 
@@ -429,18 +487,6 @@ def test_get_linode_types_overrides(test_linode_client):
         assert linode_type.region_prices[0].monthly >= 0
 
 
-def test_get_linode_type_by_id(test_linode_client):
-    pytest.skip(
-        "Might need Type to match how other object models are behaving e.g. client.load(Type, 123)"
-    )
-
-
-def test_get_linode_type_gpu():
-    pytest.skip(
-        "Might need Type to match how other object models are behaving e.g. client.load(Type, 123)"
-    )
-
-
 def test_save_linode_noforce(test_linode_client, create_linode):
     linode = create_linode
     old_label = linode.label
@@ -464,28 +510,29 @@ def test_save_linode_force(test_linode_client, create_linode):
 
 
 class TestNetworkInterface:
-    def test_list(self, create_linode):
-        linode = create_linode
+    def test_list(self, linode_for_network_interface_tests):
+        linode = linode_for_network_interface_tests
 
         config: Config = linode.configs[0]
 
         config.interface_create_public(
             primary=True,
         )
-        config.interface_create_vlan(
-            label="testvlan", ipam_address="10.0.0.3/32"
-        )
+
+        label = str(time.time_ns()) + "vlabel"
+
+        config.interface_create_vlan(label=label, ipam_address="10.0.0.3/32")
 
         interface = config.network_interfaces
 
         assert interface[0].purpose == "public"
         assert interface[0].primary
         assert interface[1].purpose == "vlan"
-        assert interface[1].label == "testvlan"
+        assert interface[1].label == label
         assert interface[1].ipam_address == "10.0.0.3/32"
 
-    def test_create_public(self, create_linode):
-        linode = create_linode
+    def test_create_public(self, linode_for_network_interface_tests):
+        linode = linode_for_network_interface_tests
 
         config: Config = linode.configs[0]
 
@@ -502,8 +549,8 @@ def test_create_public(self, create_linode):
         assert interface.purpose == "public"
         assert interface.primary
 
-    def test_create_vlan(self, create_linode):
-        linode = create_linode
+    def test_create_vlan(self, linode_for_network_interface_tests):
+        linode = linode_for_network_interface_tests
 
         config: Config = linode.configs[0]
 
@@ -521,7 +568,11 @@ def test_create_vlan(self, create_linode):
         assert interface.label == "testvlan"
         assert interface.ipam_address == "10.0.0.2/32"
 
-    def test_create_vpc(self, create_linode, create_vpc_with_subnet_and_linode):
+    def test_create_vpc(
+        self,
+        linode_for_network_interface_tests,
+        create_vpc_with_subnet_and_linode,
+    ):
         vpc, subnet, linode, _ = create_vpc_with_subnet_and_linode
 
         config: Config = linode.configs[0]
@@ -545,7 +596,11 @@ def test_create_vpc(self, create_linode, create_vpc_with_subnet_and_linode):
         assert interface.ipv4.nat_1_1 == linode.ipv4[0]
         assert interface.ip_ranges == ["10.0.0.5/32"]
 
-    def test_update_vpc(self, create_linode, create_vpc_with_subnet_and_linode):
+    def test_update_vpc(
+        self,
+        linode_for_network_interface_tests,
+        create_vpc_with_subnet_and_linode,
+    ):
         vpc, subnet, linode, _ = create_vpc_with_subnet_and_linode
 
         config: Config = linode.configs[0]
@@ -575,25 +630,31 @@ def test_update_vpc(self, create_linode, create_vpc_with_subnet_and_linode):
         assert interface.ipv4.nat_1_1 == linode.ipv4[0]
         assert interface.ip_ranges == ["10.0.0.6/32"]
 
-    def test_reorder(self, create_linode):
-        linode = create_linode
+    def test_reorder(self, linode_for_network_interface_tests):
+        linode = linode_for_network_interface_tests
 
         config: Config = linode.configs[0]
 
         pub_interface = config.interface_create_public(
             primary=True,
         )
+
+        label = str(time.time_ns()) + "vlabel"
         vlan_interface = config.interface_create_vlan(
-            label="testvlan", ipam_address="10.0.0.3/32"
+            label=label, ipam_address="10.0.0.3/32"
         )
 
+        send_request_when_resource_available(300, linode.shutdown)
+
         interfaces = config.network_interfaces
         interfaces.reverse()
 
-        config.interface_reorder(interfaces)
+        send_request_when_resource_available(
+            300, config.interface_reorder, interfaces
+        )
         config.invalidate()
 
-        assert [v.id for v in config.interfaces] == [
+        assert [v.id for v in config.interfaces[:2]] == [
             vlan_interface.id,
             pub_interface.id,
         ]
@@ -606,7 +667,11 @@ def test_delete_interface_containing_vpc(
         config: Config = linode.configs[0]
 
         config.interfaces = []
-        config.save()
+
+        # must power off linode before saving
+        send_request_when_resource_available(300, linode.shutdown)
+
+        send_request_when_resource_available(60, config.save)
 
         interface = config.interface_create_vpc(
             subnet=subnet,