Add test workflow with unittest (#189)

* Add `.gitignore`

* Add github test workflow

* Add push trigger

* Add expected failures and skips

* Also test on python version 3.11

* Remove python version 3.11 due to dependency error

* Add note on how to run tests

* Add `tf-keras` as a dependency

* Add expect failure for several tests and test classes all suffering from `AttributeError: module 'tensorflow_model_analysis' has no attribute 'EvalConfig'`

These are to be addressed in a future PR

* Remove import to nonexistant modules

* Install `libprotobuf-c-dev` for unit tests in CI

* Temporarily remove `expectedFailure`s

* Fix `EvalConfig` imports

* Add more `expectedFailure`s

* Remove `unexpectedFailure` from unexpected success

* Add Python 3.11 to CI tests

* Remove `libprotobuf-c-dev`

* Remove unnecessary import

* Add `expectedFailure`

* Use `skip` instead of `expectedFailure`

* Fix code comment

* Run for all users

---------

Co-authored-by: Peyton Murray <[email protected]>

Author: smokestacklightnin

.github/workflows/ci-test.yml

@@ -0,0 +1,42 @@
+# Github action definitions for unit-tests with PRs.
+
+name: tfma-unit-tests
+on:
+  push:
+  pull_request:
+    branches: [ master ]
+    paths-ignore:
+      - '**.md'
+      - 'docs/**'
+  workflow_dispatch:
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: ['3.9', '3.10', '3.11']
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+        cache-dependency-path: |
+          setup.py
+
+    - name: Install dependencies
+      run: |
+        sudo apt update
+        sudo apt install -y protobuf-compiler
+        pip install .
+
+    - name: Run unit tests
+      shell: bash
+      run: |
+        python -m unittest discover -p "*_test.py"

README.md

@@ -85,6 +85,16 @@ cd dist
 pip3 install tensorflow_model_analysis-<version>-py3-none-any.whl
 ```
 
+### Running tests
+
+To run tests, run 
+
+```
+python -m unittest discover -p *_test.py
+```
+
+from the root project directory.
+
 ### Jupyter Lab
 
 As of writing, because of https://github.com/pypa/pip/issues/9187, `pip install`

setup.py

@@ -342,6 +342,7 @@ def select_constraint(default, nightly=None, git_master=None):
             nightly='>=1.18.0.dev',
             git_master='@git+https://github.com/tensorflow/tfx-bsl@master',
         ),
+        'tf-keras',
     ],
     'extras_require': {
         'all': [*_make_extra_packages_tfjs(), *_make_docs_packages()],

tensorflow_model_analysis/api/model_eval_lib_test.py

@@ -16,6 +16,7 @@
 import json
 import os
 import tempfile
+import unittest
 
 from absl.testing import absltest
 from absl.testing import parameterized
@@ -1122,6 +1123,9 @@ def testRunModelAnalysisWithQueryBasedMetrics(self):
       for k in expected_metrics[group]:
         self.assertIn(k, got_metrics[group])
 
+  # PR 189: Remove the `skip` mark if the test passes for all supported versions
+  # of python
+  @unittest.skip('Fails for some versions of Python, including 3.9')
   def testRunModelAnalysisWithUncertainty(self):
     examples = [
         self._makeExample(age=3.0, language='english', label=1.0),
@@ -1391,6 +1395,8 @@ def testRunModelAnalysisWithSchema(self):
     self.assertEqual(1.0, got_buckets[1]['lowerThresholdInclusive'])
     self.assertEqual(2.0, got_buckets[-2]['upperThresholdExclusive'])
 
+  # PR 189: Remove the `expectedFailure` mark if the test passes
+  @unittest.expectedFailure
   def testLoadValidationResult(self):
     result = validation_result_pb2.ValidationResult(validation_ok=True)
     path = os.path.join(absltest.get_default_test_tmpdir(), 'results.tfrecord')
@@ -1399,6 +1405,8 @@ def testLoadValidationResult(self):
     loaded_result = model_eval_lib.load_validation_result(path)
     self.assertTrue(loaded_result.validation_ok)
 
+  # PR 189: Remove the `expectedFailure` mark if the test passes
+  @unittest.expectedFailure
   def testLoadValidationResultDir(self):
     result = validation_result_pb2.ValidationResult(validation_ok=True)
     path = os.path.join(
@@ -1409,6 +1417,8 @@ def testLoadValidationResultDir(self):
     loaded_result = model_eval_lib.load_validation_result(os.path.dirname(path))
     self.assertTrue(loaded_result.validation_ok)
 
+  # PR 189: Remove the `expectedFailure` mark if the test passes
+  @unittest.expectedFailure
   def testLoadValidationResultEmptyFile(self):
     path = os.path.join(
         absltest.get_default_test_tmpdir(), constants.VALIDATIONS_KEY

tensorflow_model_analysis/export_only/__init__.py

@@ -29,5 +29,3 @@ def eval_input_receiver_fn():
   tfma_export.export.export_eval_saved_model(...)
 """
 
-from tensorflow_model_analysis.eval_saved_model import export
-from tensorflow_model_analysis.eval_saved_model import exporter

tensorflow_model_analysis/extractors/inference_base_test.py

@@ -34,6 +34,8 @@
 from tensorflow_serving.apis import logging_pb2
 from tensorflow_serving.apis import prediction_log_pb2
 
+import unittest
+
 
 class TfxBslPredictionsExtractorTest(testutil.TensorflowModelAnalysisTest):
 
@@ -70,6 +72,8 @@ def _create_tfxio_and_feature_extractor(
     )
     return tfx_io, feature_extractor
 
+  # PR 189: Remove the `expectedFailure` mark if the test passes
+  @unittest.expectedFailure
   def testIsValidConfigForBulkInferencePass(self):
     saved_model_proto = text_format.Parse(
         """
@@ -129,6 +133,8 @@ def testIsValidConfigForBulkInferencePass(self):
         )
     )
 
+  # PR 189: Remove the `expectedFailure` mark if the test passes
+  @unittest.expectedFailure
   def testIsValidConfigForBulkInferencePassDefaultSignatureLookUp(self):
     saved_model_proto = text_format.Parse(
         """
@@ -184,6 +190,8 @@ def testIsValidConfigForBulkInferencePassDefaultSignatureLookUp(self):
         )
     )
 
+  # PR 189: Remove the `expectedFailure` mark if the test passes
+  @unittest.expectedFailure
   def testIsValidConfigForBulkInferenceFailNoSignatureFound(self):
     saved_model_proto = text_format.Parse(
         """
@@ -239,6 +247,8 @@ def testIsValidConfigForBulkInferenceFailNoSignatureFound(self):
         )
     )
 
+  # PR 189: Remove the `expectedFailure` mark if the test passes
+  @unittest.expectedFailure
   def testIsValidConfigForBulkInferenceFailKerasModel(self):
     saved_model_proto = text_format.Parse(
         """
@@ -296,6 +306,8 @@ def testIsValidConfigForBulkInferenceFailKerasModel(self):
         )
     )
 
+  # PR 189: Remove the `expectedFailure` mark if the test passes
+  @unittest.expectedFailure
   def testIsValidConfigForBulkInferenceFailWrongInputType(self):
     saved_model_proto = text_format.Parse(
         """

tensorflow_model_analysis/metrics/bleu_test.py

@@ -19,6 +19,7 @@
 import numpy as np
 import tensorflow as tf
 import tensorflow_model_analysis as tfma
+from tensorflow_model_analysis.proto import config_pb2
 from tensorflow_model_analysis import constants
 from tensorflow_model_analysis.evaluators import metrics_plots_and_validations_evaluator
 from tensorflow_model_analysis.metrics import bleu
@@ -573,7 +574,7 @@ def test_bleu_end_2_end(self):
           }
         }
         """,
-        tfma.EvalConfig(),
+        config_pb2.EvalConfig(),
     )
 
     example1 = {

tensorflow_model_analysis/metrics/example_count_test.py

@@ -19,6 +19,7 @@
 import numpy as np
 import tensorflow as tf
 import tensorflow_model_analysis as tfma
+from tensorflow_model_analysis.proto import config_pb2
 from tensorflow_model_analysis.metrics import example_count
 from tensorflow_model_analysis.metrics import metric_types
 from tensorflow_model_analysis.metrics import metric_util
@@ -109,7 +110,7 @@ def testExampleCountsWithoutLabelPredictions(self):
           }
         }
         """,
-        tfma.EvalConfig(),
+        config_pb2.EvalConfig(),
     )
     name_list = ['example_count']
     expected_results = [0.6]

tensorflow_model_analysis/metrics/object_detection_confusion_matrix_metrics_test.py

@@ -18,10 +18,10 @@
 from apache_beam.testing import util
 import numpy as np
 import tensorflow_model_analysis as tfma
+from tensorflow_model_analysis.proto import config_pb2
 from tensorflow_model_analysis.metrics import metric_types
 from google.protobuf import text_format
 
-
 class ObjectDetectionConfusionMatrixMetricsTest(parameterized.TestCase):
 
   @parameterized.named_parameters(('_max_recall',
@@ -41,7 +41,7 @@ class ObjectDetectionConfusionMatrixMetricsTest(parameterized.TestCase):
                    '"max_num_detections":100, "name":"maxrecall"'
           }
         }
-        """, tfma.EvalConfig()), ['maxrecall'], [2 / 3]),
+        """, config_pb2.EvalConfig()), ['maxrecall'], [2 / 3]),
                                   ('_precision_at_recall',
                                    text_format.Parse(
                                        """
@@ -59,7 +59,7 @@ class ObjectDetectionConfusionMatrixMetricsTest(parameterized.TestCase):
                    '"max_num_detections":100, "name":"precisionatrecall"'
           }
         }
-        """, tfma.EvalConfig()), ['precisionatrecall'], [3 / 5]),
+        """, config_pb2.EvalConfig()), ['precisionatrecall'], [3 / 5]),
                                   ('_recall',
                                    text_format.Parse(
                                        """
@@ -77,7 +77,7 @@ class ObjectDetectionConfusionMatrixMetricsTest(parameterized.TestCase):
                    '"max_num_detections":100, "name":"recall"'
           }
         }
-        """, tfma.EvalConfig()), ['recall'], [2 / 3]), ('_precision',
+        """, config_pb2.EvalConfig()), ['recall'], [2 / 3]), ('_precision',
                                                         text_format.Parse(
                                                             """
         model_specs {
@@ -94,7 +94,7 @@ class ObjectDetectionConfusionMatrixMetricsTest(parameterized.TestCase):
                    '"max_num_detections":100, "name":"precision"'
           }
         }
-        """, tfma.EvalConfig()), ['precision'], [0.5]), ('_threshold_at_recall',
+        """, config_pb2.EvalConfig()), ['precision'], [0.5]), ('_threshold_at_recall',
                                                          text_format.Parse(
                                                              """
         model_specs {
@@ -111,7 +111,7 @@ class ObjectDetectionConfusionMatrixMetricsTest(parameterized.TestCase):
                    '"max_num_detections":100, "name":"thresholdatrecall"'
           }
         }
-        """, tfma.EvalConfig()), ['thresholdatrecall'], [0.3]))
+        """, config_pb2.EvalConfig()), ['thresholdatrecall'], [0.3]))
   def testObjectDetectionMetrics(self, eval_config, name_list,
                                  expected_results):
 

tensorflow_model_analysis/metrics/object_detection_confusion_matrix_plot_test.py

@@ -18,6 +18,7 @@
 from apache_beam.testing import util
 import numpy as np
 import tensorflow_model_analysis as tfma
+from tensorflow_model_analysis.proto import config_pb2
 from tensorflow_model_analysis.metrics import metric_types
 from tensorflow_model_analysis.utils import test_util
 
@@ -45,7 +46,7 @@ def testConfusionMatrixPlot(self):
                    '"max_num_detections":100, "name":"iou0.5"'
           }
         }
-        """, tfma.EvalConfig())
+        """, config_pb2.EvalConfig())
     extracts = [
         # The match at iou_threshold = 0.5 is
         # gt_matches: [[0]] dt_matches: [[0, -1]]