maintainable tests mechanism added

Author: Mohamed-Ashraf273

.github/workflows/actions.yml

@@ -63,13 +63,7 @@ jobs:
         pip install keras-nightly --progress-bar off
     - name: Test with pytest
       run: |
-        if [ "${{ matrix.backend }}" == "openvino" ]; then
-          IGNORE_FILE="openvino_excluded_tests.txt"
-          IGNORE_ARGS=$(awk '{print "--ignore=" $0}' "$IGNORE_FILE")
-        else
-          IGNORE_ARGS=""
-        fi
-        pytest keras_hub/ $IGNORE_ARGS
+        pytest keras_hub/
     - name: Run integration tests
       run: |
         python pip_build.py --install

conftest.py

@@ -3,6 +3,9 @@
 import keras
 import pytest
 
+from keras_hub.src.utils.openvino_utils import get_openvino_skip_reason
+from keras_hub.src.utils.openvino_utils import setup_openvino_test_config
+
 
 def pytest_addoption(parser):
     parser.addoption(
@@ -29,6 +32,13 @@ def pytest_addoption(parser):
         default=False,
         help="fail if a gpu is not present",
     )
+    parser.addoption(
+        "--auto_skip_training",
+        action="store_true",
+        default=True,
+        help="automatically skip tests with "
+        "training methods on non-trainable backends",
+    )
 
 
 def pytest_configure(config):
@@ -70,16 +80,13 @@ def pytest_configure(config):
         "markers",
         "kaggle_key_required: mark test needing a kaggle key",
     )
-    config.addinivalue_line(
-        "markers",
-        "requires_trainable_backend: mark test for trainable backend only",
-    )
 
 
 def pytest_collection_modifyitems(config, items):
     run_extra_large_tests = config.getoption("--run_extra_large")
     # Run large tests for --run_extra_large or --run_large.
     run_large_tests = config.getoption("--run_large") or run_extra_large_tests
+    auto_skip_training = config.getoption("--auto_skip_training")
 
     # Messages to annotate skipped tests with.
     skip_large = pytest.mark.skipif(
@@ -114,43 +121,23 @@ def pytest_collection_modifyitems(config, items):
         if "kaggle_key_required" in item.keywords:
             item.add_marker(kaggle_key_required)
 
-    openvino_skipped_tests = []
-    if keras.config.backend() == "openvino":
-        from pathlib import Path
-
-        workspace_root = Path(__file__).resolve().parents[0]
-        file_path = workspace_root / "openvino_excluded_concrete_tests.txt"
-        with open(file_path, "r") as file:
-            openvino_skipped_tests = [
-                line.strip() for line in file if line.strip()
-            ]
-
-    requires_trainable_backend = pytest.mark.skipif(
-        keras.config.backend() in ["openvino"],
-        reason="fit not implemented for OpenVINO backend.",
-    )
-
-    for item in items:
-        if "requires_trainable_backend" in item.keywords:
-            item.add_marker(requires_trainable_backend)
-        # also, skip concrete tests for openvino, listed in the special file
-        # this is more granular mechanism to exclude tests rather
-        # than using --ignore option
-        for skipped_test in openvino_skipped_tests:
-            if skipped_test in item.nodeid:
+        # OpenVINO-specific skipping logic - whitelist-based approach
+        if keras.config.backend() == "openvino":
+            # OpenVINO backend configuration
+            from pathlib import Path
+
+            openvino_supported_paths = setup_openvino_test_config(
+                str(Path(__file__).parent)
+            )
+            skip_reason = get_openvino_skip_reason(
+                item,
+                openvino_supported_paths,
+                auto_skip_training,
+            )
+            if skip_reason:
                 item.add_marker(
-                    skip_if_backend(
-                        "openvino",
-                        "Not supported operation by openvino backend",
-                    )
+                    pytest.mark.skipif(True, reason=f"OpenVINO: {skip_reason}")
                 )
-                break
-
-
-def skip_if_backend(given_backend, reason):
-    return pytest.mark.skipif(
-        keras.config.backend() == given_backend, reason=reason
-    )
 
 
 # Disable traceback filtering for quicker debugging of tests failures.

integration_tests/basic_usage_test.py

@@ -6,10 +6,6 @@
 import keras_hub
 
 
-@unittest.skipIf(
-    keras.backend.backend() == "openvino",
-    "Skip for non-trainable backends like OpenVINO",
-)
 class BasicUsageTest(unittest.TestCase):
     def test_transformer(self):
         # Tokenize some inputs with a binary label.

keras_hub/src/layers/modeling/alibi_bias_test.py

@@ -1,5 +1,4 @@
 import keras
-import pytest
 from keras import ops
 from keras import random
 
@@ -8,7 +7,6 @@
 
 
 class AlibiBiasTest(TestCase):
-    @pytest.mark.requires_trainable_backend
     def test_layer_behaviors(self):
         alibi_bias_max = 8
         batch_size = 4

keras_hub/src/layers/modeling/anchor_generator_test.py

@@ -14,7 +14,6 @@
     reason="Bbox utils are not supported before keras < 3.8.0",
 )
 class AnchorGeneratorTest(TestCase):
-    @pytest.mark.requires_trainable_backend
     def test_layer_behaviors(self):
         images_shape = (8, 128, 128, 3)
         self.run_layer_test(

keras_hub/src/layers/modeling/cached_multi_head_attention_test.py

@@ -1,4 +1,3 @@
-import pytest
 from keras import ops
 from keras import random
 
@@ -9,7 +8,6 @@
 
 
 class CachedMultiHeadAttentionTest(TestCase):
-    @pytest.mark.requires_trainable_backend
     def test_layer_behaviors(self):
         self.run_layer_test(
             cls=CachedMultiHeadAttention,
@@ -77,7 +75,6 @@ def call(outputs, cache):
         self.assertAllClose(output, no_loop_outputs)
         self.assertAllClose(output_cache, no_loop_cache)
 
-    @pytest.mark.requires_trainable_backend
     def test_training_propagation(self):
         batch_size = 2
         seq_len = 5

keras_hub/src/layers/modeling/f_net_encoder_test.py

@@ -1,4 +1,3 @@
-import pytest
 from keras import ops
 from keras import random
 
@@ -7,7 +6,6 @@
 
 
 class FNetEncoderTest(TestCase):
-    @pytest.mark.requires_trainable_backend
     def test_layer_behaviors(self):
         self.run_layer_test(
             cls=FNetEncoder,
@@ -33,7 +31,6 @@ def test_value_error_when_invalid_kernel_initializer(self):
                 kernel_initializer="Invalid",
             )
 
-    @pytest.mark.requires_trainable_backend
     def test_training_propagation(self):
         x = random.uniform(shape=(2, 4, 6))
         layer = FNetEncoder(

keras_hub/src/layers/modeling/masked_lm_head_test.py

@@ -1,4 +1,3 @@
-import pytest
 from keras import random
 
 from keras_hub.src.layers.modeling.masked_lm_head import MaskedLMHead
@@ -9,7 +8,6 @@
 
 
 class MaskedLMHeadTest(TestCase):
-    @pytest.mark.requires_trainable_backend
     def test_layer_behaviors(self):
         self.run_layer_test(
             cls=MaskedLMHead,
@@ -29,7 +27,6 @@ def test_layer_behaviors(self):
             expected_num_trainable_weights=6,
         )
 
-    @pytest.mark.requires_trainable_backend
     def test_layer_behaviors_with_embedding(self):
         embedding = ReversibleEmbedding(100, 16)
         embedding.build((4, 10))

keras_hub/src/layers/modeling/position_embedding_test.py

@@ -1,6 +1,5 @@
 import keras
 import numpy as np
-import pytest
 from keras import ops
 from keras import random
 
@@ -16,7 +15,6 @@ def custom_init(shape, dtype=None):
 
 
 class PositionEmbeddingTest(TestCase):
-    @pytest.mark.requires_trainable_backend
     def test_layer_behaviors(self):
         self.run_layer_test(
             cls=PositionEmbedding,
@@ -28,7 +26,6 @@ def test_layer_behaviors(self):
             expected_num_trainable_weights=1,
         )
 
-    @pytest.mark.requires_trainable_backend
     def test_layer_behaviors_4d(self):
         self.run_layer_test(
             cls=PositionEmbedding,

keras_hub/src/layers/modeling/reversible_embedding_test.py

@@ -2,7 +2,6 @@
 
 import keras
 import numpy as np
-import pytest
 from absl.testing import parameterized
 from keras import ops
 from keras import random
@@ -18,7 +17,6 @@ class ReversibleEmbeddingTest(TestCase):
         ("tie_weights", True),
         ("untie_weights", False),
     )
-    @pytest.mark.requires_trainable_backend
     def test_layer_behaviors_tied(self, tie_weights):
         self.run_layer_test(
             cls=ReversibleEmbedding,