Merge pull request #55 from Mbed-TLS/dev/gilles-peskine-arm/analyze_outcome-split-framework

Framework: Split check_test_cases.py and outcome_analysis.py

Author: gilles-peskine-arm

scripts/check_test_cases.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+
+"""Sanity checks for test data."""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+
+import argparse
+import re
+import sys
+
+from mbedtls_framework import collect_test_cases
+
+
+class DescriptionChecker(collect_test_cases.TestDescriptionExplorer):
+    """Check all test case descriptions.
+
+* Check that each description is valid (length, allowed character set, etc.).
+* Check that there is no duplicated description inside of one test suite.
+"""
+
+    def __init__(self, results):
+        self.results = results
+
+    def new_per_file_state(self):
+        """Dictionary mapping descriptions to their line number."""
+        return {}
+
+    def process_test_case(self, per_file_state,
+                          file_name, line_number, description):
+        """Check test case descriptions for errors."""
+        results = self.results
+        seen = per_file_state
+        if description in seen:
+            results.error(file_name, line_number,
+                          'Duplicate description (also line {})',
+                          seen[description])
+            return
+        if re.search(br'[\t;]', description):
+            results.error(file_name, line_number,
+                          'Forbidden character \'{}\' in description',
+                          re.search(br'[\t;]', description).group(0).decode('ascii'))
+        if re.search(br'[^ -~]', description):
+            results.error(file_name, line_number,
+                          'Non-ASCII character in description')
+        if len(description) > 66:
+            results.warning(file_name, line_number,
+                            'Test description too long ({} > 66)',
+                            len(description))
+        seen[description] = line_number
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--list-all',
+                        action='store_true',
+                        help='List all test cases, without doing checks')
+    parser.add_argument('--quiet', '-q',
+                        action='store_true',
+                        help='Hide warnings')
+    parser.add_argument('--verbose', '-v',
+                        action='store_false', dest='quiet',
+                        help='Show warnings (default: on; undoes --quiet)')
+    options = parser.parse_args()
+    if options.list_all:
+        descriptions = collect_test_cases.collect_available_test_cases()
+        sys.stdout.write('\n'.join(descriptions + ['']))
+        return
+    results = collect_test_cases.Results(options)
+    checker = DescriptionChecker(results)
+    try:
+        checker.walk_all()
+    except collect_test_cases.ScriptOutputError as e:
+        results.error(e.script_name, e.idx,
+                      '"{}" should be listed as "<suite_name>;<description>"',
+                      e.line)
+    if (results.warnings or results.errors) and not options.quiet:
+        sys.stderr.write('{}: {} errors, {} warnings\n'
+                         .format(sys.argv[0], results.errors, results.warnings))
+    sys.exit(1 if results.errors else 0)
+
+if __name__ == '__main__':
+    main()

scripts/mbedtls_framework/collect_test_cases.py

@@ -0,0 +1,166 @@
+"""Discover all the test cases (unit tests and SSL tests)."""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+
+import glob
+import os
+import re
+import subprocess
+import sys
+
+from . import build_tree
+
+
+class ScriptOutputError(ValueError):
+    """A kind of ValueError that indicates we found
+    the script doesn't list test cases in an expected
+    pattern.
+    """
+
+    @property
+    def script_name(self):
+        return super().args[0]
+
+    @property
+    def idx(self):
+        return super().args[1]
+
+    @property
+    def line(self):
+        return super().args[2]
+
+class Results:
+    """Store file and line information about errors or warnings in test suites."""
+
+    def __init__(self, options):
+        self.errors = 0
+        self.warnings = 0
+        self.ignore_warnings = options.quiet
+
+    def error(self, file_name, line_number, fmt, *args):
+        sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n').
+                         format(file_name, line_number, *args))
+        self.errors += 1
+
+    def warning(self, file_name, line_number, fmt, *args):
+        if not self.ignore_warnings:
+            sys.stderr.write(('{}:{}:Warning:' + fmt + '\n')
+                             .format(file_name, line_number, *args))
+            self.warnings += 1
+
+class TestDescriptionExplorer:
+    """An iterator over test cases with descriptions.
+
+The test cases that have descriptions are:
+* Individual unit tests (entries in a .data file) in test suites.
+* Individual test cases in ssl-opt.sh.
+
+This is an abstract class. To use it, derive a class that implements
+the process_test_case method, and call walk_all().
+"""
+
+    def process_test_case(self, per_file_state,
+                          file_name, line_number, description):
+        """Process a test case.
+
+per_file_state: an object created by new_per_file_state() at the beginning
+                of each file.
+file_name: a relative path to the file containing the test case.
+line_number: the line number in the given file.
+description: the test case description as a byte string.
+"""
+        raise NotImplementedError
+
+    def new_per_file_state(self):
+        """Return a new per-file state object.
+
+The default per-file state object is None. Child classes that require per-file
+state may override this method.
+"""
+        #pylint: disable=no-self-use
+        return None
+
+    def walk_test_suite(self, data_file_name):
+        """Iterate over the test cases in the given unit test data file."""
+        in_paragraph = False
+        descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
+        with open(data_file_name, 'rb') as data_file:
+            for line_number, line in enumerate(data_file, 1):
+                line = line.rstrip(b'\r\n')
+                if not line:
+                    in_paragraph = False
+                    continue
+                if line.startswith(b'#'):
+                    continue
+                if not in_paragraph:
+                    # This is a test case description line.
+                    self.process_test_case(descriptions,
+                                           data_file_name, line_number, line)
+                in_paragraph = True
+
+    def collect_from_script(self, script_name):
+        """Collect the test cases in a script by calling its listing test cases
+option"""
+        descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
+        listed = subprocess.check_output(['sh', script_name, '--list-test-cases'])
+        # Assume test file is responsible for printing identical format of
+        # test case description between --list-test-cases and its OUTCOME.CSV
+        #
+        # idx indicates the number of test case since there is no line number
+        # in the script for each test case.
+        for idx, line in enumerate(listed.splitlines()):
+            # We are expecting the script to list the test cases in
+            # `<suite_name>;<description>` pattern.
+            script_outputs = line.split(b';', 1)
+            if len(script_outputs) == 2:
+                suite_name, description = script_outputs
+            else:
+                raise ScriptOutputError(script_name, idx, line.decode("utf-8"))
+
+            self.process_test_case(descriptions,
+                                   suite_name.decode('utf-8'),
+                                   idx,
+                                   description.rstrip())
+
+    @staticmethod
+    def collect_test_directories():
+        """Get the relative path for the TLS and Crypto test directories."""
+        mbedtls_root = build_tree.guess_mbedtls_root()
+        directories = [os.path.join(mbedtls_root, 'tests'),
+                       os.path.join(mbedtls_root, 'tf-psa-crypto', 'tests')]
+        directories = [os.path.relpath(p) for p in directories]
+        return directories
+
+    def walk_all(self):
+        """Iterate over all named test cases."""
+        test_directories = self.collect_test_directories()
+        for directory in test_directories:
+            for data_file_name in glob.glob(os.path.join(directory, 'suites',
+                                                         '*.data')):
+                self.walk_test_suite(data_file_name)
+
+            for sh_file in ['ssl-opt.sh', 'compat.sh']:
+                sh_file = os.path.join(directory, sh_file)
+                if os.path.isfile(sh_file):
+                    self.collect_from_script(sh_file)
+
+class TestDescriptions(TestDescriptionExplorer):
+    """Collect the available test cases."""
+
+    def __init__(self):
+        super().__init__()
+        self.descriptions = set()
+
+    def process_test_case(self, _per_file_state,
+                          file_name, _line_number, description):
+        """Record an available test case."""
+        base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name))
+        key = ';'.join([base_name, description.decode('utf-8')])
+        self.descriptions.add(key)
+
+def collect_available_test_cases():
+    """Collect the available test cases."""
+    explorer = TestDescriptions()
+    explorer.walk_all()
+    return sorted(explorer.descriptions)