diff --git a/tools/bin/mbedtls-prepare-build b/tools/bin/mbedtls-prepare-build index 38860d8..97f3585 100755 --- a/tools/bin/mbedtls-prepare-build +++ b/tools/bin/mbedtls-prepare-build @@ -2,22 +2,54 @@ """Generate a makefile for Mbed Crypto or Mbed TLS. """ +import abc import argparse +import collections +import enum import glob import itertools import os import platform import re +import shlex import shutil import subprocess import sys import tempfile +import typing +from typing import Dict, Iterable, List, Optional, Set, Tuple + + +T = typing.TypeVar('T') +V = typing.TypeVar('V') + +# The typing_extensions module is necessary for type annotations that are +# checked with mypy. It is only used for type annotations or to define +# things that are themselves only used for type annotations. It is not +# available on a default Python <3.8 installation. Therefore, try loading +# what we need from it for the sake of mypy (which depends on, or comes +# with, typing_extensions), and if not define substitutes that lack the +# static type information but are good enough at runtime. +try: + from typing_extensions import Protocol #pylint: disable=import-error +except ImportError: + class Protocol: #type: ignore + #pylint: disable=too-few-public-methods + pass + +class Writable(Protocol): + """Abstract class for typing hints.""" + # pylint: disable=no-self-use,too-few-public-methods,unused-argument + def write(self, text: str) -> typing.Any: + ... + -def sjoin(*args): +def sjoin(*args: str) -> str: """Join the arguments (strings) with a single space between each.""" return ' '.join(args) -def append_to_value(d, key, *values): +def append_to_value(d: Dict[T, List[V]], + key: T, *values: V) -> None: """Append to a value in a dictionary. Append values to d[key]. Create an empty list if d[key] does not exist. @@ -25,7 +57,12 @@ def append_to_value(d, key, *values): lst = d.setdefault(key, []) lst += values -def are_same_existing_files(*files): +def are_same_existing_files(*files: str) -> bool: + """Whether all the given files are the same file. + + Symbolic links are considered the same as their target. + Non-existent paths are not considered the same as anything, even themselves. + """ for file1 in files: if not os.path.exists(file1): return False @@ -34,6 +71,37 @@ def are_same_existing_files(*files): return False return True +def open_write_safe(filename: str, + temp_filename: Optional[str] = None, + **kwargs) -> typing.IO[str]: + # TODO: this function should be a protocol class that creates a temporary + # file and renames it when closing on success. + """Open the given file for writing, but only if it looks generated. + + If temp_filename is given, open temp_filename for writing, but expect + the content to go into filename eventually. + + The goal of this function is to only allow overwriting files that were + generated by a previous run of this script. + + Not protected against race conditions. + """ + if os.path.exists(filename): + with open(filename, 'r') as inp: + line = inp.readline() + if 'Generated by' not in line: + raise Exception('Not overwriting {} which looks hand-written' + .format(filename)) + if temp_filename is not None and os.path.exists(temp_filename): + with open(temp_filename, 'r') as inp: + line = inp.readline() + if not ('Generated by' in line or + (not line and temp_filename is not None)): + raise Exception('Not overwriting {} which looks hand-written' + .format(temp_filename)) + return open(filename if temp_filename is None else temp_filename, 'w') + + class EnvironmentOption: """A description of options that set makefile variables. @@ -45,8 +113,9 @@ class EnvironmentOption: * default: a default value if the variable is not in the environment. """ - def __init__(self, var, default='', help=None, - option=None): + def __init__(self, var: str, default='', + help: Optional[str] = None, + option: Optional[str] = None) -> None: self.var = var self.attr = var self.option = ('--' + var.lower().replace('_', '-') @@ -85,6 +154,8 @@ _environment_options = [ 'Options to pass to ${CC} when linking a program that uses threads'), EnvironmentOption('LIBRARY_EXTRA_CFLAGS', '', 'Options to pass to ${CC} when compiling library sources'), + EnvironmentOption('LIBTESTDRIVER1_EXTRA_CFLAGS', '', + 'Options to pass to ${CC} when compiling libtestdriver1'), EnvironmentOption('PERL', 'perl', 'Perl interpreter'), EnvironmentOption('PROGRAMS_EXTRA_CFLAGS', '', @@ -117,12 +188,18 @@ _environment_options = [ 'Options to always pass to ${CC}'), ] + +# For typing purposes. To be refined. +Options = argparse.Namespace + + """The list of potential submodules. A submodule is a subdirectory of the source tree which has the same general structure as the source tree. """ -_submodule_names = ['crypto'] +SUBMODULE_NAMES = ['crypto', 'tf-psa-crypto'] + class SourceFile: """A description of a file path in the source tree. @@ -132,44 +209,44 @@ class SourceFile: not in a submodule), and the path inside the submodule. """ - def __init__(self, root, submodule, inner_path): + def __init__(self, root: str, submodule: str, inner_path: str) -> None: self.root = root self.submodule = submodule self.inner_path = inner_path - def sort_key(self): + def sort_key(self) -> Tuple[str, bool, str]: # Compare by inner path first, then by submodule. # The empty submodule comes last. return (self.inner_path, not self.submodule, self.submodule) - def __lt__(self, other): + def __lt__(self, other: 'SourceFile') -> bool: if self.root != other.root: raise TypeError("Cannot compare source files under different roots" , self, other) return self.sort_key() < other.sort_key() - def relative_path(self): + def relative_path(self) -> str: """Path to the file from the root of the source tree.""" return os.path.join(self.submodule, self.inner_path) - def source_dir(self): + def source_dir(self) -> str: """Path to the directory containing the file, from the root of the source tree.""" return os.path.dirname(self.relative_path()) - def real_path(self): + def real_path(self) -> str: """A path at which the file can be opened during makefile generation.""" return os.path.join(self.root, self.submodule, self.inner_path) - def make_path(self): + def make_path(self) -> str: """A path to the file that is valid in the makefile.""" if self.submodule: return '/'.join(['$(SOURCE_DIR)', self.submodule, self.inner_path]) else: return '$(SOURCE_DIR)/' + self.inner_path - def target_dir(self): + def target_dir(self) -> str: """The target directory for build products of this source file. This is the path to the directory containing the source file @@ -177,11 +254,11 @@ class SourceFile: """ return os.path.dirname(self.inner_path) - def base(self): + def base(self) -> str: """The path to the file inside the submodule, without the extension.""" return os.path.splitext(self.inner_path)[0] - def target(self, extension): + def target(self, extension) -> str: """A build target for this source file, with the specified extension.""" return self.base() + extension @@ -192,35 +269,61 @@ class GeneratedFile(SourceFile): inside the build tree rather than a file inside the source tree. """ - def __init__(self, path): + def __init__(self, path: str) -> None: super().__init__('.', '', path) - def make_path(self): + def make_path(self) -> str: return self.inner_path -class ClassicTestGenerator: + +class TestGeneratorInterface: + """Test generator script description interface.""" + + __metaclass__ = abc.ABCMeta + + @abc.abstractmethod + def target(self, c_file: str) -> str: + """A space-separated list of generated files.""" + ... + + @abc.abstractmethod + def script(self, source_dir: str) -> str: + """The path to the generator script.""" + ... + + @abc.abstractmethod + def function_files(self, function_file: str) -> List[str]: + """The full list of .function files used to generate the given suite. + + This always includes the argument, and can include other files + (helpers.function, etc.). + """ + ... + + @abc.abstractmethod + def command(self, function_file: str, data_file: str) -> str: + """The build command to generate the given test suite.""" + ... + +class ClassicTestGenerator(TestGeneratorInterface): """Test generator script description for the classic (<<2.13) test generator (generate_code.pl).""" - def __init__(self, options): + def __init__(self, options: Options) -> None: self.options = options - @staticmethod - def target(c_file): + def target(self, c_file: str) -> str: return c_file - @staticmethod - def script(_source_dir): + def script(self, _source_dir: str) -> str: return 'tests/scripts/generate_code.pl' - @staticmethod - def function_files(function_file): + def function_files(self, function_file: str) -> List[str]: return ['tests/suites/helpers.function', 'tests/suites/main_test.function', function_file] - @staticmethod - def command(function_file, data_file): + def command(self, function_file: str, data_file: str) -> str: source_dir = os.path.dirname(function_file) if source_dir != os.path.dirname(data_file): raise Exception('Function file and data file are not in the same directory', @@ -237,41 +340,56 @@ class ClassicTestGenerator: os.path.splitext(os.path.basename(function_file))[0], os.path.splitext(os.path.basename(data_file))[0]) -class OnTargetTestGenerator: +class OnTargetTestGenerator(TestGeneratorInterface): """Test generator script description for the >=2.13 test generator with on-target testing support (generate_test_code.py).""" - def __init__(self, options): + def __init__(self, options: Options) -> None: self.options = options - @staticmethod - def target(c_file): + def target(self, c_file: str) -> str: datax_file = os.path.splitext(c_file)[0] + '.datax' return sjoin(c_file, datax_file) - @staticmethod - def script(source_dir): - return os.path.dirname(source_dir) + '/scripts/generate_test_code.py' + def script(self, source_dir: str) -> str: + if os.path.exists(os.path.join(self.options.source, source_dir, + '../scripts/generate_test_code.py')): + return os.path.dirname(source_dir) + '/scripts/generate_test_code.py' + else: + return 'framework/scripts/generate_test_code.py' - @staticmethod - def function_files(function_file, on_target=False): + def helpers_dir(self, source_dir: str) -> str: + if os.path.exists(os.path.join(self.options.source, + source_dir, + 'helpers.function')): + return source_dir + elif os.path.exists(os.path.join(self.options.source, + 'tf-psa-crypto', + source_dir, + 'helpers.function')): + # tf-psa-crypto transition + return 'tf-psa-crypto/' + source_dir + else: + raise Exception('Unable to locate test helpers for ' + source_dir) + + def function_files(self, function_file: str, on_target=False) -> List[str]: source_dir = os.path.dirname(function_file) - return (['{}/{}.function'.format(source_dir, helper) + return (['{}/{}.function'.format(self.helpers_dir(source_dir), helper) for helper in ['helpers', 'main_test', 'target_test' if on_target else 'host_test']] + [function_file]) - @classmethod - def command(cls, function_file, data_file): + def command(self, function_file: str, data_file: str) -> str: source_dir = os.path.dirname(function_file) suite_path = '$(SOURCE_DIR_FROM_TESTS)/' + source_dir + helpers_path = '$(SOURCE_DIR_FROM_TESTS)/' + self.helpers_dir(source_dir) return sjoin('$(PYTHON)', - '$(SOURCE_DIR_FROM_TESTS)/' + cls.script(source_dir), + '$(SOURCE_DIR_FROM_TESTS)/' + self.script(source_dir), '-f $(SOURCE_DIR_FROM_TESTS)/' + function_file, '-d $(SOURCE_DIR_FROM_TESTS)/' + data_file, - '-t', suite_path + '/main_test.function', - '-p', suite_path + '/host_test.function', - '--helpers-file', suite_path + '/helpers.function', + '-t', helpers_path + '/main_test.function', + '-p', helpers_path + '/host_test.function', + '--helpers-file', helpers_path + '/helpers.function', '-s', suite_path, '-o .') @@ -291,7 +409,25 @@ class MakefileMaker: 'library/psa_crypto_driver_wrappers.h', ]) - def __init__(self, options, source_path): + @staticmethod + def gather_submodules(source_dir) -> Iterable[str]: + """Iterate over the existing submodules under source_dir. + + 3rdparty directories are considered submodules, as well as + names registered in SUBMODULE_NAMES. + """ + for m in SUBMODULE_NAMES: + if os.path.isdir(os.path.join(source_dir, m)): + yield m + for d in glob.glob(os.path.join(source_dir, '3rdparty', '*')): + continue #TODO + if os.path.isdir(os.path.join(d, 'include')) or \ + os.path.isdir(os.path.join(d, 'library')) or \ + os.path.isdir(os.path.join(d, 'programs')) or \ + os.path.isdir(os.path.join(d, 'tests')): + yield d[len(source_dir)+1:] + + def __init__(self, options: Options, source_path: str) -> None: """Initialize a makefile generator. options is the command line option object. @@ -313,11 +449,11 @@ class MakefileMaker: self.object_extension = self.options.object_extension self.shared_library_extension = self.options.shared_library_extension self.source_path = source_path - self.out = None - self.static_libraries = None - self.help = {} - self.clean = [] - self.variables_to_export = set() + self.out = None #type: Optional[Writable] + self.static_libraries = None #type: Optional[List[str]] + self.help = {} #type: Dict[str, str] # target: help_text + self.clean = [] #type: List[str] # file names or patterns to clean + self.variables_to_export = set() #type: Set[str] if options.QEMU_LD_PREFIX: self.variables_to_export.add('QEMU_LD_PREFIX') self.dependency_cache = { @@ -326,20 +462,22 @@ class MakefileMaker: # dependencies change over time. 'library/psa_crypto_driver_wrappers.c': self.PSA_CRYPTO_DRIVER_WRAPPERS_DEPENDENCIES, } - self.submodules = [submodule for submodule in _submodule_names - if self.source_exists(submodule)] - if self.source_exists('tests/scripts/generate_test_code.py'): - self.test_generator = OnTargetTestGenerator(options) + self.submodules = list(self.gather_submodules(self.options.source)) + if self.source_exists('tests/scripts/generate_test_code.py') or \ + self.source_exists('framework/scripts/generate_test_code.py'): + self.test_generator = OnTargetTestGenerator(options) #type: TestGeneratorInterface else: self.test_generator = ClassicTestGenerator(options) # Unset fields that are only meaningful at certain later times. # Setting them here makes Pylint happy, but having set them here # makes it harder to diagnose if some method is buggy and attempts # to use a field whose value isn't actually known. + # (TODO: this was back before mypy. Now it's probably simpler to + # allow None, since mypy forces to assert 'is not None' explicitly?) del self.static_libraries # Set when generating the library targets del self.out # Set only while writing the output file - def get_file_submodule(self, filename): + def get_file_submodule(self, filename: str) -> Tuple[Optional[str], str]: """Break up a path into submodule and inner path. More precisely, given a path filename from the root of the source @@ -355,19 +493,20 @@ class MakefileMaker: return submodule, filename[len(submodule) + 1:] return None, filename - def crypto_file_path(self, filename): + def crypto_file_path(self, filename: str) -> str: """Return the path to a crypto file. Look for the file at the given path in the crypto submodule, and if it exists, return its path from the root of the source tree. Otherwise return filename unchanged. """ - in_crypto = os.path.join('crypto', filename) - if os.path.exists(in_crypto): - filename = in_crypto + for submodule in SUBMODULE_NAMES: + in_submodule = os.path.join(submodule, filename) + if os.path.exists(in_submodule): + filename = in_submodule return '$(SOURCE_DIR)/' + filename - def source_exists(self, filename): + def source_exists(self, filename: str) -> bool: """Test if the given path exists in the source tree. This function does not try different submodules. If the file is @@ -375,15 +514,16 @@ class MakefileMaker: """ return os.path.exists(os.path.join(self.options.source, filename)) - def line(self, text): + def line(self, text: str) -> None: """Emit a makefile line.""" + assert self.out is not None self.out.write(text + '\n') - def words(self, *words): + def words(self, *words: str) -> None: """Emit a makefile line obtain by joining the words with spaces.""" self.line(' '.join(words)) - def assign(self, name, *value_words): + def assign(self, name: str, *value_words: str) -> None: """Emit a makefile line that contains an assignment. The assignment is to the variable called name, and its value @@ -392,6 +532,9 @@ class MakefileMaker: nonempty_words = [word for word in value_words if word] self.line(' '.join([name, '='] + nonempty_words)) + # TODO: what should be the type of format? Mypy can check format strings + # since https://github.com/python/mypy/pull/7418, but how do I indicate + # that template is a formatting template? def format(self, template, *args): """Emit a makefile line containing the given formatted template.""" self.line(template.format(*args)) @@ -400,7 +543,7 @@ class MakefileMaker: """Emit a makefile comment line containing the given formatted template.""" self.format('## ' + template, *args) - def add_dependencies(self, name, *dependencies): + def add_dependencies(self, name: str, *dependencies: str) -> None: """Generate dependencies for name.""" parts = (name + ':',) + dependencies simple = ' '.join(parts) @@ -411,32 +554,32 @@ class MakefileMaker: _glob2re_re = re.compile(r'[.^$*+?{}\|()]') @staticmethod - def _glob2re_repl(m): + def _glob2re_repl(m: typing.Match[str]) -> str: if m.group(0) == '*': - return '[^/]*' + return r'[^/]*' elif m.group(0) == '?': - return '[^/]' + return r'[^/]' else: return '\\' + m.group(0) @classmethod - def glob2re(cls, pattern): + def glob2re(cls, pattern: str) -> str: # Simple glob pattern to regex translator. Does not support # character sets properly, which is ok because we don't use them. # We don't use fnmatch or PurePath.match because they let # '*' and '?' match '/'. return re.sub(cls._glob2re_re, cls._glob2re_repl, pattern) - def is_already_cleaned(self, name): + def is_already_cleaned(self, name: str) -> bool: if not self.clean: return False - regex = ''.join(['\A(?:', + regex = ''.join([r'\A(?:', '|'.join([self.glob2re(pattern) for patterns in self.clean for pattern in patterns.split()]), - ')\Z']) - return re.match(regex, name) + r')\Z']) + return re.match(regex, name) is not None - def add_clean(self, *elements): + def add_clean(self, *elements: str) -> None: """Add one or more element to the list of things to clean. These can be file paths or wildcard patterns. They can contain @@ -447,8 +590,12 @@ class MakefileMaker: """ self.clean.append(' '.join(elements)) - def target(self, name, dependencies, commands, - help=None, phony=False, clean=None, short=None): + def target(self, name: str, dependencies: Iterable[str], + commands: Iterable[str], + help: Optional[str] = None, + phony=False, + clean: Optional[bool] = None, + short: Optional[str] = None) -> None: """Generate a makefile rule. * name: the target(s) of the rule. This is a string. If there are @@ -490,7 +637,7 @@ class MakefileMaker: if clean: self.add_clean(name) - def setenv_command(self): + def setenv_command(self) -> str: """Generate a shell command to export some environment variables. The values of these variables must not contain the character ``'`` @@ -505,7 +652,7 @@ class MakefileMaker: for name in sorted(self.variables_to_export)]) + '; ') - def environment_option_subsection(self): + def environment_option_subsection(self) -> None: """Generate the assignments to customizable options.""" self.comment('Tool settings') for envopt in _environment_options: @@ -514,7 +661,7 @@ class MakefileMaker: self.assign(envopt.var, getattr(self.options, envopt.attr)) - def settings_section(self): + def settings_section(self) -> None: """Generate assignments to customizable and internal variables. Some additional section-specified variables are assigned in each @@ -557,7 +704,7 @@ class MakefileMaker: self.assign('SOURCE_DIR_FROM_TESTS', '../$(SOURCE_DIR)') self.assign('VALGRIND_LOG_DIR_FROM_TESTS', '.') - def include_path(self, filename): + def include_path(self, filename: str) -> List[str]: """Return the include path for filename. filename must be a path relative to the root of the source tree. @@ -566,7 +713,21 @@ class MakefileMaker: """ dirs = [] submodule, base = self.get_file_submodule(filename) - subdirs = ['include', 'include/mbedtls', 'library'] + # When there is a tf-psa-crypto submodule, we need the toplevel + # include in addition to the one in the subdmodule. The calls + # to list_source_files only return matches from the submodule if + # present. + subdirs = ['include'] + for pattern in [ + 'include', + 'drivers/*/include', + 'library', + 'core', + 'drivers/*/library', + 'drivers/*/src', + ]: + matches = self.list_source_files(self.options.source, pattern) + subdirs += [source.relative_path() for source in matches] if base.startswith('tests') or base.startswith('programs'): subdirs.append('tests') if self.source_exists('tests/include'): @@ -578,14 +739,29 @@ class MakefileMaker: dirs.append(subdir) if submodule is not None: dirs.append(os.path.join(submodule, subdir)) + if submodule == '3rdparty/everest': + dirs.append(os.path.join(submodule, 'include/everest')) + if '/kremlib/' in filename: + dirs.append(os.path.join(submodule, 'include/everest/kremlib')) + elif '/everest/' in filename: + prefix = '' + if 'tf-psa-crypto/' in filename: + prefix = 'tf-psa-crypto/' + dirs.append(prefix + 'drivers/everest/include/everest') return dirs - def include_path_options(self, filename): + def include_path_options(self, filename: str) -> str: """Return the include path options (-I ...) for filename.""" + if filename.startswith('tests/libtestdriver1'): + inc = ['tests/libtestdriver1'] + if '/p256-m' in filename: + inc.append(inc[0] + '/include-testdriver1/p256-m') + return sjoin(*('-I ' + dir for dir in inc)) return ' '.join(['-I $(SOURCE_DIR)/' + dir for dir in self.include_path(filename)]) - def collect_c_dependencies(self, c_file, stack=frozenset()): + def collect_c_dependencies(self, c_file: str, + stack=frozenset()) -> typing.FrozenSet[str]: """Find the build dependencies of the specified C source file. c_file must be an existing C file in the source tree. @@ -607,14 +783,14 @@ class MakefileMaker: if c_file in self.dependency_cache: return self.dependency_cache[c_file] if c_file in stack: - return set() + return frozenset() stack |= {c_file} include_path = ([os.path.dirname(c_file)] + self.include_path(c_file)) dependencies = set() extra = set() with open(os.path.join(self.options.source, c_file)) as stream: for line in stream: - m = re.match(r'#include ["<](.*)[">]', line) + m = re.match(r' *# *include *["<](.*?)[">]', line) if m is None: continue filename = m.group(1) @@ -629,10 +805,11 @@ class MakefileMaker: for dep in frozenset(dependencies): dependencies |= self.collect_c_dependencies(dep, stack) dependencies |= extra - self.dependency_cache[c_file] = dependencies - return dependencies + frozen = frozenset(dependencies) + self.dependency_cache[c_file] = frozen + return frozen - def is_generated(self, filename): + def is_generated(self, filename: str) -> bool: """Whether the specified C file is generated. Implemented with heuristics based on the name. @@ -642,6 +819,8 @@ class MakefileMaker: """ if 'ssl_debug_helpers_generated.' in filename: return False + if 'value_names_generated' in filename: + return False if '_generated.' in filename: return True if 'psa_crypto_driver_wrappers.c' in filename: @@ -650,17 +829,21 @@ class MakefileMaker: #return True return False - def is_include_only(self, filename): + INCLUDE_ONLY_C_FILES = frozenset([ + 'FStar_UInt128_extracted.c', # included by Hacl_Curve25519_joined.c + 'FStar_UInt64_FStar_UInt32_FStar_UInt16_FStar_UInt8.c', # included by Hacl_Curve25519_joined.c + 'Hacl_Curve25519.c', # included by Hacl_Curve25519_joined.c + 'psa_constant_names_generated.c', # included by psa_constant_names.c + 'ssl_test_common_source.c', # included by ssl_*2.c + ]) + def is_include_only(self, filename: str) -> bool: """Whether the specified C file is only meant for use in "#include" directive. Implemented with heuristics based on the name. """ - return os.path.basename(filename) in { - 'psa_constant_names_generated.c', - 'ssl_test_common_source.c', - } + return os.path.basename(filename) in self.INCLUDE_ONLY_C_FILES - def c_with_dependencies(self, c_file): + def c_with_dependencies(self, c_file: str) -> List[str]: """A list of C dependencies in makefile syntax. Generate the depdendencies of c_file with collect_c_dependencies, @@ -677,7 +860,7 @@ class MakefileMaker: '$(SOURCE_DIR)/' + filename) for filename in sorted(deps) + [c_file]] - def c_dependencies_only(self, c_files): + def c_dependencies_only(self, c_files: Iterable[str]) -> List[str]: """A list of C dependencies in makefile syntax. Generate the depdendencies of each element of c_files with @@ -687,13 +870,25 @@ class MakefileMaker: The elements of c_files themselves are included not in the resulting list unless they are a dependency of another element. """ - deps = set.union(*[self.collect_c_dependencies(c_file) - for c_file in c_files]) + deps = frozenset.union(*[self.collect_c_dependencies(c_file) + for c_file in c_files]) return ['$(SOURCE_DIR)/' + filename for filename in sorted(deps)] - def object_target(self, section, src, extra): - c_file = src.make_path() - dependencies = self.c_with_dependencies(src.relative_path()) + def object_target(self, section: str, + src: SourceFile, + deps: Iterable[str] = (), + auto_deps: bool = True, + extra_flags: str = '') -> None: + c_path = src.make_path() + dependencies = list(deps) + if auto_deps: + dependencies += self.c_with_dependencies(src.relative_path()) + if self.options.accel_list: + # TODO: refine dependencies (same as in + # libtestdriver1_source_targets()) + dependencies.append('$(libtestdriver1_headers)') + else: + dependencies += [c_path] for short, switch, extension in [ ('CC -S ', '-S', self.assembly_extension), ('CC ', '-c', self.object_extension), @@ -704,16 +899,41 @@ class MakefileMaker: '$(WARNING_CFLAGS)', '$(COMMON_FLAGS)', '$(CFLAGS)', - ' '.join(['$({}_CFLAGS)'.format(section)] + - extra), + '$({}_CFLAGS)'.format(section), + # TODO: The section CFLAGS already contain some + # -I options. But in the library, we need + # extra -I options for some files (e.g. everest). + # So we add partially redundant -I options here. + self.include_path_options(c_path), + extra_flags.strip(), '-o $@', - switch, c_file)], - short=(short + c_file)) + switch, c_path)], + short=(short + c_path)) - _potential_libraries = ['crypto', 'x509', 'tls'] + class Library(enum.Enum): + TESTDRIVER1 = 1 + CRYPTO = 2 + X509 = 3 + TLS = 4 - @staticmethod - def library_of(module): + def __lt__(self, other: 'MakefileMaker.Library') -> bool: + return self.value < other.value + + def core(self) -> str: + return self.name.lower() + + def libname(self) -> str: + if self == self.TESTDRIVER1: + return 'libtestdriver1' + return 'libmbed' + self.core() + + # For tracing when a library is accidentally implicitly converted to a string + def __str__(self): + import pdb; pdb.set_trace() + return super().__str__() + + @classmethod + def library_of(cls, module: str) -> Library: """Identify which Mbed TLS library contains the specified module. This function bases the result on known module names, defaulting @@ -721,36 +941,244 @@ class MakefileMaker: """ module = os.path.basename(module) if module.startswith('x509') or \ - module in ['certs', 'pkcs11']: - return 'x509' + module in ['certs', 'pkcs7', 'pkcs11']: + return cls.Library.X509 elif module.startswith('ssl') or \ module in ['debug', 'net', 'net_sockets']: - return 'tls' + return cls.Library.TLS else: - return 'crypto' + return cls.Library.CRYPTO - @staticmethod - def dash_l_lib(lib): + def dash_l_lib(self, lib: typing.Union[Library, str]) -> str: """Return the -l option to link with the specified library.""" - base = os.path.splitext(os.path.basename(lib))[0] + if isinstance(lib, self.Library): + base = lib.libname() + else: + base = os.path.splitext(os.path.basename(lib))[0] if base.startswith('lib'): base = base[3:] - if not base.startswith('mbed'): - base = 'mbed' + base return '-l' + base - def psa_crypto_driver_wrappers_subsection(self, contents): + def build_libtestdriver1_rewrite_code(self) -> Optional[str]: + """Construct the perl code to rewrite libtestdriver1. + + Pass this code to ``perl -p -e``. + """ + return '; '.join([r's[(\x23 *include *["<])((?:everest|mbedtls|p256-m|psa)/)][\1include-testdriver1/\2]', + r'next if /\x23 *include\b/', + r's/\b(?=(?:mbedtls|psa)_)/libtestdriver1_/g', + r's/\b(?=(?:MBEDTLS|PSA)_)/LIBTESTDRIVER1_/g']) + + CONFIG_TEST_DRIVER_H = 'tests/include/test/drivers/config_test_driver.h' + CONFIG_TEST_DRIVER_EXTENSION_H = 'tests/include/test/drivers/crypto_config_test_driver_extension.h' + + def libtestdriver1_c_target(self, src: SourceFile) -> Optional[str]: + """Generate a target for a libtestdriver1 C source file. + + Return the path to the intermediate file within the build tree. + + We put all header files under tests/libtestdriver1/include + and all .c files under tests/libtestdriver1/library, + merging any files from submodules. + """ + short_path = src.relative_path() + basename = os.path.basename(short_path) + if basename == 'mbedtls_config.h': + # The legacy configuration is a special case, largely independent + # of the legacy configuration of the library. + # TODO: MBEDTLS_THREADING_C and MBEDTLS_THREADING_PTHREAD should + # be aligned with the library. + src = self.list_source_files(self.options.source, + self.CONFIG_TEST_DRIVER_H)[0] + if basename == 'crypto_config.h': + # The crypto configuration is a special case: it needs to have + # the library configuration with modifications described by + # CONFIG_TEST_DRIVER_EXTENSION_H. This is handled in + # test_driver_config_subsection(). + return None + elif short_path == 'library/common.h' and \ + os.path.exists(os.path.join(self.options.source, + 'tf-psa-crypto', 'core', 'common.h')): + # There is a common.h both in tf-psa-crypto and in mbedtls. + # We're building crypto files, so pick common.h from crypto. + return None + elif '/include/' in short_path: + short_path = re.sub(r'.*/include/', r'include-testdriver1/', short_path) + elif re.search(r'/p256-m/[^/]+\.h\Z', short_path): + short_path = 'include-testdriver1/p256-m/' + basename + elif short_path.startswith('include/'): + short_path = 'include-testdriver1' + short_path[7:] + else: + short_path = os.path.join('library', basename) + target_path = 'tests/libtestdriver1/' + short_path + self.target(target_path, + [src.make_path()], + [sjoin('$(PERL) -p -e', + "'$(libtestdriver1_rewrite)'", + src.make_path(), + '>$@')]) + return target_path + + def test_driver_auxiliary_files_subsection(self) -> None: + """Generate libtestdriver1 auxiliary files.""" + if os.path.exists(os.path.join(self.options.source, 'tf-psa-crypto')): + # File included by + # include/psa/crypto_driver_contexts_primitives.h + with open_write_safe(os.path.join(self.options.dir, + 'tests', 'libtestdriver1', + 'libtestdriver1', 'tf-psa-crypto', + 'include', 'psa', 'crypto.h')) as out: + out.write("""/* Generated by mbedtls-prepare-build */ +#include "../../../../include-testdriver1/psa/crypto.h" +/* End of generated file */ +""") + + def test_driver_config_subsection(self) -> None: + """Generate libtestdriver1 configuration files.""" + include_dir = os.path.join(self.options.dir, + 'tests', 'libtestdriver1', + 'include-testdriver1') + for source_path in ['include/psa/crypto_config.h', + self.CONFIG_TEST_DRIVER_EXTENSION_H]: + src = self.list_source_files(self.options.source, source_path)[0] + basename = os.path.basename(source_path) + if basename == 'crypto_config.h': + basename = 'base_crypto_config.h' + self.target('tests/libtestdriver1/include-testdriver1/psa/' + basename, + [src.make_path()], + [sjoin('$(PERL) -p -e', + "'$(libtestdriver1_rewrite)'", + src.make_path(), + '>$@')]) + with open_write_safe(os.path.join(include_dir, 'psa', 'crypto_config.h')) as out: + out.write("""/* Generated by mbedtls-prepare-build */ +#include "base_crypto_config.h" +#include "crypto_config_test_driver_extension.h" +/* End of generated file */ +""") + + def libtestdriver1_source_targets(self) -> None: + """Targets for source files in libtestdriver1. + """ + header_files = self.list_source_files(self.options.source, + 'include/*/*.h', + 'library/*.h', + 'drivers/*/*.h', + 'drivers/*/*/*.h', + 'drivers/*/*/*/*.h', + 'core/*.h') + intermediate_h_list = [] + # Add special cases from test_driver_config_subsection() + intermediate_h_list += \ + [GeneratedFile(os.path.join('tests/libtestdriver1/include-testdriver1/psa', basename)) + for basename in ['base_crypto_config.h', + os.path.basename(self.CONFIG_TEST_DRIVER_EXTENSION_H)]] + intermediate_c_list = [] + for src in header_files: + target = self.libtestdriver1_c_target(src) + if target: + intermediate_h_list.append(GeneratedFile(target)) + crypto_modules = [src + for src in self.library_modules() + if self.library_of(src.base()) == self.Library.CRYPTO] + for src in crypto_modules: + target = self.libtestdriver1_c_target(src) + if target: + intermediate_c_list.append(GeneratedFile(target)) + self.assign('libtestdriver1_headers', + sjoin(*(src.inner_path + for src in intermediate_h_list))) + self.assign('libtestdriver1_module_sources', + sjoin(*(src.inner_path + for src in intermediate_c_list))) + self.assign('libtestdriver1_modules', + sjoin(*(src.base() + for src in intermediate_c_list))) + self.assign('libtestdriver1_objects', + '$(libtestdriver1_modules:={})' + .format(self.object_extension)) + # TODO: actual dependencies .o -> .h. + # c_with_dependencies() intrinsically doesn't work on files that + # don't exist at makefile generation time. + for src in intermediate_c_list: + self.object_target('LIBTESTDRIVER1', src, + deps=['$(libtestdriver1_headers)'], + auto_deps=False) + self.target('libtestdriver1-sources', + ['$(libtestdriver1_headers)', + '$(libtestdriver1_module_sources)'], + [], + phony=True) + + def libtestdriver1_section(self) -> None: + """Generate the section of the makefile for libtestdriver1. + + The targets are object files for library modules and + static library files. + + Unlike the rest of the build, we leverage the official Makefile, + to adapt for its evolution (e.g. before/after the creation of the + framework, before/after the creation of tf-psa-crypto). + """ + self.comment('Targets for libtestdriver1') + libtestdriver1_rewrite_code = self.build_libtestdriver1_rewrite_code() + if libtestdriver1_rewrite_code is not None: + self.assign('libtestdriver1_rewrite', libtestdriver1_rewrite_code) + self.libtestdriver1_source_targets() + self.assign('LIBTESTDRIVER1_D_ACCEL', + sjoin(*('-DLIBTESTDRIVER1_MBEDTLS_PSA_ACCEL_' + accel + for accel in self.options.accel_list))) + self.assign('LIBTESTDRIVER1_CONSUMER_D_ACCEL', + sjoin(*('-DMBEDTLS_PSA_ACCEL_' + accel + for accel in self.options.accel_list))) + self.assign('LIBTESTDRIVER1_D_EXTRA', + sjoin(*('-DLIBTESTDRIVER1_MBEDTLS_PSA_ACCEL_' + accel + for accel in self.options.libtestdriver1_extra_list))) + if self.options.accel_list: + self.assign('USE_LIBTESTDRIVER1_CFLAGS', + sjoin('-I tests/libtestdriver1 -I $(SOURCE_DIR)/tests/include', + '-DPSA_CRYPTO_DRIVER_TEST', + '-DMBEDTLS_TEST_LIBTESTDRIVER1', + '$(LIBTESTDRIVER1_D_ACCEL)', + '$(LIBTESTDRIVER1_CONSUMER_D_ACCEL)')) + else: + self.assign('USE_LIBTESTDRIVER1_CFLAGS', '') + self.assign('LIBTESTDRIVER1_CFLAGS', + sjoin('$(LIBTESTDRIVER1_D_ACCEL)', + '$(LIBTESTDRIVER1_D_EXTRA)', + '$(LIBTESTDRIVER1_EXTRA_CFLAGS)')) + self.test_driver_auxiliary_files_subsection() + self.test_driver_config_subsection() + #TODO: more dependencies (adapt $(libmbedcrypt_objects)?) + prep_dependencies = [self.CONFIG_TEST_DRIVER_H, + self.CONFIG_TEST_DRIVER_EXTENSION_H] + self.target('library/libtestdriver1.a', + ['$(libtestdriver1_objects)'], + ['$(AR) $(ARFLAGS) $@ $(libtestdriver1_objects)'], + short='AR $@') + self.target('libtestdriver1', + ['library/libtestdriver1.a'], + [], + help='Build libtestdriver1.a', + phony=True) + + def psa_crypto_driver_wrappers_subsection( + self, + contents: Dict[Library, List[str]] + ) -> None: generated = 'library/psa_crypto_driver_wrappers.c' script_path = self.crypto_file_path('scripts/psa_crypto_driver_wrappers.py') sources = [self.crypto_file_path(drv) for drv in self.options.psa_driver] - contents['crypto'].append(os.path.splitext(generated)[0]) + contents[self.Library.CRYPTO].append(os.path.splitext(generated)[0]) self.target(generated, [script_path] + sources, [sjoin(script_path, '-o $@', *sources)]) - self.object_target('LIBRARY', GeneratedFile(generated), []) + self.object_target('LIBRARY', GeneratedFile(generated)) - def list_source_files(self, root, *patterns): + def list_source_files(self, + root: str, + *patterns: str) -> List[SourceFile]: """List the source files matching any of the specified patterns. Look for the specified wildcard pattern under all submodules, including @@ -762,7 +1190,7 @@ class MakefileMaker: """ # FIXME: for error.c at least, we need the root, not the submodule. all_sources = {} - for submodule in _submodule_names + ['']: + for submodule in self.submodules + ['']: submodule_root = os.path.join(root, submodule) start = len(submodule_root) if submodule: @@ -783,7 +1211,15 @@ class MakefileMaker: all_sources[base] = src return sorted(all_sources.values()) - def library_section(self): + def library_modules(self) -> Iterable[SourceFile]: + """Generate the list of C source files for the library.""" + return self.list_source_files(self.options.source, + 'core/*.c', + 'drivers/*/*.c', + 'drivers/*/*/*.c', + 'library/*.c') + + def library_section(self) -> None: """Generate the section of the makefile for the library directory. The targets are object files for library modules and @@ -794,6 +1230,7 @@ class MakefileMaker: '-I include/mbedtls', # must come first, for the config header '-I include', self.include_path_options('library/*'), + '$(USE_LIBTESTDRIVER1_CFLAGS)', '$(LIBRARY_EXTRA_CFLAGS)') self.add_clean(*['library/*' + ext for ext in (self.assembly_extension, @@ -801,29 +1238,30 @@ class MakefileMaker: self.object_extension, self.shared_library_extension)]) # Enumerate modules and emit the rules to build them - modules = self.list_source_files(self.options.source, 'library/*.c') + modules = self.library_modules() for module in modules: - self.object_target('LIBRARY', module, []) - contents = {} + self.object_target('LIBRARY', module) + contents = collections.defaultdict(list) # Enumerate libraries and the rules to build them - for lib in self._potential_libraries: - contents[lib] = [] for module in modules: contents[self.library_of(module.base())].append(module.base()) if self.options.psa_driver: self.psa_crypto_driver_wrappers_subsection(contents) - libraries = [lib for lib in self._potential_libraries - if contents[lib]] + libraries = sorted(contents.keys()) for lib in libraries: - self.format('libmbed{}_modules = {}', lib, ' '.join(contents[lib])) - self.format('libmbed{}_objects = $(libmbed{}_modules:={})', - lib, lib, self.object_extension) + libname = lib.libname() + self.format('{}_modules = {}', libname, ' '.join(contents[lib])) + self.format('{}_objects = $({}_modules:={})', + libname, libname, self.object_extension) self.static_libraries = [] + if self.options.accel_list: + self.static_libraries.append('library/libtestdriver1.a') shared_libraries = [] for idx, lib in enumerate(libraries): - objects = '$(libmbed{}_objects)'.format(lib) - static = 'library/libmbed{}{}'.format(lib, self.library_extension) - shared = 'library/libmbed{}{}'.format(lib, self.shared_library_extension) + libname = lib.libname() + objects = '$({}_objects)'.format(libname) + static = 'library/{}{}'.format(libname, self.library_extension) + shared = 'library/{}{}'.format(libname, self.shared_library_extension) self.static_libraries.append(static) shared_libraries.append(shared) self.target(static, [objects], @@ -832,14 +1270,16 @@ class MakefileMaker: dependent_libraries = libraries[:idx] if dependent_libraries: dash_l_dependent = ('-L . ' + - sjoin(*[self.dash_l_lib(lib) - for lib in dependent_libraries])) + sjoin(*[self.dash_l_lib(dep) + for dep in dependent_libraries])) else: dash_l_dependent = '' - self.target(shared, [objects] + ['library/libmbed{}{}' - .format(lib, self.shared_library_extension) - for lib in dependent_libraries], + self.target(shared, [objects] + ['library/{}{}' + .format(dep.libname(), + self.shared_library_extension) + for dep in dependent_libraries], [sjoin('$(CC)', + '$(WARNING_CFLAGS)', '$(COMMON_FLAGS)', '$(LDFLAGS)', '$(DLLFLAGS)', @@ -884,13 +1324,13 @@ class MakefileMaker: 'programs/fuzz/common', 'programs/fuzz/onefile', })) - def auxiliary_objects(self, base): + def auxiliary_objects(self, base: str) -> Iterable[str]: if base.startswith('programs/fuzz'): return ['programs/fuzz/common', 'programs/fuzz/onefile'] else: return self._auxiliary_objects.get(base, []) - def program_libraries(self, app): + def program_libraries(self, app: str) -> List[Library]: """Return the list of libraries that app uses. app is the base of the main file of a sample program (directory @@ -901,20 +1341,24 @@ class MakefileMaker: """ basename = os.path.basename(app) subdir = os.path.basename(os.path.dirname(app)) + libs = [] + if self.options.accel_list: + libs.append(self.Library.TESTDRIVER1) + libs.append(self.Library.CRYPTO) if basename == 'dlopen': return [] if (subdir == 'ssl' or basename.startswith('ssl') or subdir == 'fuzz' or basename in {'cert_app', 'dh_client', 'dh_server', 'udp_proxy'} ): - return ['crypto', 'x509', 'tls'] + return libs + [self.Library.X509, self.Library.TLS] if (subdir == 'x509' or (basename == 'selftest' and self.source_exists('library/x509.c')) ): - return ['crypto', 'x509'] - return ['crypto'] + return libs + [self.Library.X509] + return libs - def extra_link_flags(self, app): + def extra_link_flags(self, app: str) -> Iterable[str]: """Return the list of extra link flags for app. app is the base of the main file of a sample program (directory @@ -928,9 +1372,12 @@ class MakefileMaker: flags.append('$(LDFLAGS_L_THREADS)') if 'dlopen' in app: flags.append('$(LDFLAGS_L_DLOPEN)') + if 'zeroize' in app: + flags.append('-g3') return flags - def add_run_target(self, program, executable=None): + def add_run_target(self, program: str, + executable: Optional[str] = None) -> None: if executable is None: executable = program + self.executable_extension self.target(program + '.run', @@ -942,7 +1389,8 @@ class MakefileMaker: ['$(SETENV)$(RUN) ' + executable + ' $(RUNS)', 'mv gmon.out $@']) - def program_subsection(self, src, executables): + def program_subsection(self, src: SourceFile, + executables: List[str]) -> None: """Emit the makefile rules for the given sample program. src is a SourceFile object refering to a source file under programs/. @@ -965,44 +1413,61 @@ class MakefileMaker: [script_path], short='Gen $@') self.add_clean(base + '_generated.c') - extra_includes = ['-I', src.target_dir()] # for generated files + extra_flags = '-I ' + src.target_dir() # for generated files + if 'zeroize' in base: + extra_flags += ' -g3' object_file = src.target(self.object_extension) - self.object_target('PROGRAMS', src, extra_includes) + self.object_target('PROGRAMS', src, extra_flags=extra_flags) if base in self._auxiliary_sources: return exe_file = src.target(self.executable_extension) object_deps = [dep + self.object_extension for dep in self.auxiliary_objects(base) if self.source_exists(dep + '.c')] - if base == 'programs/test/dlopen': - object_deps.append('library/platform.o library/platform_util.o') - else: - object_deps.append('$(test_common_objects)') libs = list(reversed(self.program_libraries(base))) - lib_files = ['library/libmbed{}{}'.format(lib, self.library_extension) + lib_files = ['library/{}{}'.format(lib.libname(), self.library_extension) for lib in libs] dash_l_libs = [self.dash_l_lib(lib) for lib in libs] + if base == 'programs/test/dlopen': + for base in ['platform', 'platform_util']: + for lib in ['library', 'drivers/builtin/src']: + if self.source_exists(os.path.join(lib, base) + '.c'): + object_deps.append(os.path.join(lib, base) + self.object_extension) + break + if self.source_exists(os.path.join('tf-psa-crypto', lib, base) + '.c'): + object_deps.append(os.path.join(lib, base) + self.object_extension) + break + else: + object_deps.append('$(test_crypto_objects)') + if self.Library.X509 in libs: + object_deps.append('$(test_x509_objects)') + if self.Library.TLS in libs: + object_deps.append('$(test_ssl_objects)') self.target(exe_file, [object_file] + object_deps + lib_files, [sjoin('$(CC)', object_file, sjoin(*object_deps), + '$(WARNING_CFLAGS)', '$(COMMON_FLAGS)', '$(LDFLAGS)', '$(PROGRAMS_LDFLAGS)', - sjoin(*(dash_l_libs + self.extra_link_flags(base))), + sjoin(*(dash_l_libs + + list(self.extra_link_flags(base)))), '$(EXTRA_LIBS)', '-o $@')], clean=False, short='LD $@') executables.append(exe_file) + self.add_run_target(src.base()) - def programs_section(self): + def programs_section(self) -> None: """Emit the makefile rules to build the sample programs.""" self.comment('Sample programs') self.assign('PROGRAMS_CFLAGS', '-I include', self.include_path_options('programs/*/*'), + '$(USE_LIBTESTDRIVER1_CFLAGS)', '$(PROGRAMS_EXTRA_CFLAGS)') self.assign('PROGRAMS_LDFLAGS', '-L library', @@ -1011,7 +1476,7 @@ class MakefileMaker: for ext in (self.assembly_extension, self.object_extension)]) programs = self.list_source_files(self.options.source, 'programs/*/*.c') - executables = [] + executables = [] #type: List[str] for src in programs: self.program_subsection(src, executables) dirs = set(src.target_dir() for src in programs) @@ -1025,11 +1490,22 @@ class MakefileMaker: help='Build the sample programs.', phony=True) self.add_clean('$(programs)') - self.add_run_target('programs/test/benchmark') - self.add_run_target('programs/test/selftest') + self.target('ssl-opt', + [program for program in executables + if program.startswith('programs/ssl')] + + ['programs/test/query_compile_time_config$(EXEXT)', + 'programs/test/udp_proxy$(EXEXT)', + 'programs/ssl/seedfile', + 'tests/seedfile'], + [], + help='Build the programs needed for ssl-opt.sh', + phony=True) # TODO: *_demo.sh + self.target('test_zeroize', + ['programs/test/zeroize'], + ['gdb -x ../tests/scripts/test_zeroize.gdb -nw -batch -nx']) - def define_tests_common_objects(self): + def define_tests_common_objects(self) -> None: """Emit the definition of tests_common_objects. These objects are needed for unit tests and for sample programs @@ -1038,15 +1514,31 @@ class MakefileMaker: """ tests_common_sources = self.list_source_files(self.options.source, 'tests/src/*.c', - 'tests/src/drivers/*.c') - tests_common_objects = [] + 'tests/src/drivers/*.c', + 'tests/src/test_helpers/*.c') + tests_crypto_objects = [] + tests_x509_objects = [] + tests_ssl_objects = [] for src in tests_common_sources: - self.object_target('TESTS', src, []) + self.object_target('TESTS', src) object_file = src.target(self.object_extension) - tests_common_objects.append(object_file) - self.assign('test_common_objects', *tests_common_objects) - - def test_subsection(self, src, executables, groups): + if 'ssl' in object_file: + tests_ssl_objects.append(object_file) + elif 'x509' in object_file: + tests_x509_objects.append(object_file) + else: + tests_crypto_objects.append(object_file) + self.assign('test_crypto_objects', *tests_crypto_objects) + self.assign('test_x509_objects', *tests_x509_objects) + self.assign('test_ssl_objects', *tests_ssl_objects) + self.assign('test_common_objects', + '$(test_crypto_objects)', + '$(test_x509_objects)', + '$(test_ssl_objects)') + + def test_subsection(self, src: SourceFile, + executables: List[str], + groups: Dict[str, List[str]]): """Emit the makefile rules to build one test suite. src is a SourceFile object for a .data file. @@ -1070,6 +1562,7 @@ class MakefileMaker: function_file = os.path.join(source_dir, function_base + '.function') function_files = self.test_generator.function_files(function_file) c_file = os.path.join('tests', base + '.c') + object_file = os.path.join('tests', base + self.object_extension) exe_basename = base + self.executable_extension exe_file = os.path.join('tests', exe_basename) if function_base in groups: @@ -1082,20 +1575,22 @@ class MakefileMaker: [data_file])], ['cd tests && ' + generate_command], short='Gen $@') + self.object_target('TESTS', GeneratedFile(c_file), + auto_deps=False, + deps=self.c_with_dependencies(function_file), + extra_flags='$(TESTS_CFLAGS)') self.target(exe_file, (self.c_dependencies_only(function_files) + - ['$(lib)', '$(test_build_deps)', c_file]), + ['$(lib)', '$(test_build_deps)', object_file]), [sjoin('$(CC)', + object_file, '$(WARNING_CFLAGS)', '$(COMMON_FLAGS)', - '$(CFLAGS)', - '$(TESTS_CFLAGS)', - '$(TESTS_EXTRA_OBJECTS)', - c_file, '$(LDFLAGS)', '$(TESTS_LDFLAGS)', '$(test_common_objects)', '$(test_libs)', + '$(TESTS_EXTRA_OBJECTS)', '-o $@')], clean=False, short='CC $@') @@ -1127,42 +1622,43 @@ class MakefileMaker: short='VALGRIND tests/' + exe_basename, phony=True) - def test_group_targets(self, groups): - """Emit run targets for test groups. + def test_group_target(self, base : str, executables : List[str]) -> None: + """Emit run targets for a test group. A test group is a group of test executables that share the same - .function file. The groups parameter is a dictionary mapping group - names to the list of executables that they contain. + .function file. """ use_run_test_suites = False - for base, executables in groups.items(): - shell_code = ''' + shell_code = ''' failures=; for x in {}; do -./$$x || failures="$$failures $$x"; +$(RUN) ./$$x $(RUNS) || failures="$$failures $$x"; done; if [ -n "$$failures" ]; then echo; echo "Failures:$$failures"; false; fi '''.format(' '.join([re.sub(r'.*/', r'', exe) for exe in executables])) - self.target('tests/' + base + '.run', - groups[base] + ['tests/seedfile'], - ['$(SETENV)cd tests && ' + shell_code], - short='', - phony=True) + self.target('tests/' + base + '.run', + executables + ['tests/seedfile'], + ['$(SETENV)cd tests && ' + shell_code], + short='', + phony=True) - def tests_section(self): + def tests_section(self) -> None: """Emit makefile rules to build and run test suites.""" self.comment('Test targets') self.assign('TESTS_CFLAGS', '-Wno-unused-function', '-I include', self.include_path_options('tests/*'), + '$(USE_LIBTESTDRIVER1_CFLAGS)', '$(TESTS_EXTRA_CFLAGS)') self.assign('TESTS_LDFLAGS', '-L library', '$(TESTS_EXTRA_LDFLAGS)') self.assign('TESTS_EXTRA_OBJECTS') + assert self.static_libraries is not None self.assign('test_libs', - *[self.dash_l_lib(lib) for lib in reversed(self.static_libraries)], + *[self.dash_l_lib(lib) + for lib in reversed(self.static_libraries)], '$(EXTRA_LIBS)') self.assign('test_build_deps', '$(test_common_objects)', *self.static_libraries) @@ -1173,12 +1669,18 @@ if [ -n "$$failures" ]; then echo; echo "Failures:$$failures"; false; fi self.add_clean('tests/*.c', 'tests/*.datax') data_files = self.list_source_files(self.options.source, 'tests/suites/*.data') - executables = [] - groups = {} + executables = [] #type: List[str] + groups = {} #type: Dict[str, List[str]] for src in data_files: self.test_subsection(src, executables, groups) self.assign('test_apps', *executables) - self.test_group_targets(groups) + for base in sorted(groups.keys()): + # If there's both a foo.data and a foo.bar.data, the + # foo.run targets runs the foo executable, and we can't reuse + # that name for a group. + if 'tests/' + base in executables: + continue + self.test_group_target(base, groups[base]) self.target('tests', ['$(test_apps)'], [], help='Build the host tests.', @@ -1216,7 +1718,7 @@ if [ -n "$$failures" ]; then echo; echo "Failures:$$failures"; false; fi rm -f files.info tests.info all.info final.info descriptions """ - def misc_section(self): + def misc_section(self) -> None: """Emit miscellaneous other targets. """ if self.options.coverage_targets: @@ -1225,24 +1727,41 @@ if [ -n "$$failures" ]; then echo; echo "Failures:$$failures"; false; fi [line.strip() for line in self.COVERAGE_RULE.split('\n')], help='Generate test coverage report') - def help_lines(self): + def help_lines(self) -> Iterable[str]: """Return the lines of text to show for the 'help' target.""" return ['{:<14} : {}'.format(name, self.help[name]) for name in sorted(self.help.keys())] - def variables_help_lines(self): + def variables_help_lines(self) -> Iterable[str]: """Return the lines of text to show for the 'help-variables' target.""" env_opts = [(envopt.var, envopt.help) - for envopt in _environment_options] + for envopt in _environment_options + if envopt.help is not None] ad_hoc = [ ('V', 'Show commands in full if non-empty.') ] return ['{:<14} # {}'.format(name, text) for (name, text) in sorted(env_opts + ad_hoc)] - def output_all(self): + QUOTE_TO_DELAY_RE = re.compile(r'(\')(--[-0-9A-Z_a-z]+=)') + def quote_arg_for_shell(self, arg) -> str: + """Quote a string for use in the platform's shell.""" + quoted = shlex.quote(arg) + # Be a bit nicer: `'--foo=hello world'` -> `--foo='hello world'` + quoted = self.QUOTE_TO_DELAY_RE.sub(r'\2\1', quoted) + return quoted + + def generation_command_for_shell(self) -> str: + """The shell command used to create the build tree.""" + return ' '.join(self.quote_arg_for_shell(arg) for arg in sys.argv) + + def generation_command_for_make(self) -> str: + """The shell command used to create the build tree, quoted for make.""" + return self.generation_command_for_shell().replace('$', '$$') + + def output_all(self) -> None: """Emit the whole makefile.""" - self.comment('Generated by {}', ' '.join(sys.argv)) + self.comment('Generated by ' + self.generation_command_for_shell()) self.comment('Do not edit this file! All modifications will be lost.') self.line('') self.settings_section() @@ -1254,8 +1773,17 @@ if [ -n "$$failures" ]; then echo; echo "Failures:$$failures"; false; fi help='Build the library, the tests and the sample programs.', phony=True) self.line('') + self.target('prepare', [], + ['cd $(SOURCE_DIR) && ' + + self.generation_command_for_make()], + help='Regenerate this makefile.', + phony=True) + self.target('dep', ['prepare'], [], phony=True) + self.line('') self.target('pwd', [], ['pwd'], phony=True, short='PWD') # for testing self.line('') + self.libtestdriver1_section() + self.line('') self.library_section() self.line('') self.define_tests_common_objects() @@ -1271,6 +1799,14 @@ if [ -n "$$failures" ]; then echo; echo "Failures:$$failures"; false; fi help='Remove all generated files.', short='RM {generated files}', phony=True) + self.target('libtestdriver1-clean', [], + ['$(RM) ' + patterns + for patterns in self.clean + if patterns.startswith('tests/libtestdriver1/')] + + ['$(RM) library/libtestdriver1.a'], + help='Remove libtestdriver1 source and object files.', + short='RM tests/libtestdriver1/** library/libtestdriver1.a', + phony=True) self.line('') self.target('help-variables', [], ['@echo "{}"'.format(line) for line in self.variables_help_lines()], @@ -1286,11 +1822,11 @@ if [ -n "$$failures" ]; then echo; echo "Failures:$$failures"; false; fi self.line('') self.comment('End of generated file.') - def generate(self): + def generate(self) -> None: """Generate the makefile.""" destination = os.path.join(self.options.dir, 'Makefile') temp_file = destination + '.new' - with open(temp_file, 'w') as out: + with open_write_safe(destination, temp_file) as out: try: self.out = out self.output_all() @@ -1298,13 +1834,14 @@ if [ -n "$$failures" ]; then echo; echo "Failures:$$failures"; false; fi del self.out os.replace(temp_file, destination) + class ConfigMaker: """Parent class for config.h or mbedtls_config.h builders. Typical usage: ChildClass(options).run() """ - def __init__(self, options): + def __init__(self, options: Options) -> None: """Initialize a config header builder with the given command line options.""" self.options = options self.source_file = options.config_file @@ -1325,30 +1862,30 @@ class ConfigMaker: self.target_file = os.path.join(options.dir, 'include', 'mbedtls', basename) - def start(self): + def start(self) -> None: """Builder-specific method which is called first.""" raise NotImplementedError - def set(self, name, value=None): + def set(self, name: str, value: Optional[str] = None) -> None: """Builder-specific method to set name to value.""" raise Exception("Configuration method {} does not support setting options" - .format(options.config_mode)) + .format(self.options.config_mode)) - def unset(self, name): + def unset(self, name: str) -> None: """Builder-specific method to unset name.""" raise Exception("Configuration method {} does not support unsetting options" - .format(options.config_mode)) + .format(self.options.config_mode)) - def batch(self, name): + def batch(self, name: str) -> None: """Builder-specific method to set the configuration with the given name.""" raise Exception("Configuration method {} does not support batch-setting options" - .format(options.config_mode)) + .format(self.options.config_mode)) - def finish(self): + def finish(self) -> None: """Builder-specific method which is called last.""" raise NotImplementedError - def run(self): + def run(self) -> None: """Go ahead and generate the config header.""" self.start() if self.options.config_name is not None: @@ -1371,16 +1908,16 @@ class ConfigMaker: value = spec[m.end('sep'):] self.set(name, value) self.finish() -_config_classes = {} +_config_classes = {} #type: Dict[str, typing.Type[ConfigMaker]] class ConfigCopy(ConfigMaker): """ConfigMaker implementation that copies the config header and runs the config script.""" - def start(self): + def start(self) -> None: if not are_same_existing_files(self.source_file, self.target_file): shutil.copyfile(self.source_file, self.target_file) - def run_config_script(self, *args): + def run_config_script(self, *args: str) -> None: if os.path.exists(os.path.join(self.options.source, 'scripts', 'config.py')): cmd = [sys.executable, 'scripts/config.py'] @@ -1389,47 +1926,70 @@ class ConfigCopy(ConfigMaker): cmd += ['-f', os.path.abspath(self.target_file)] + list(args) subprocess.check_call(cmd, cwd=self.options.source) - def set(self, name, value=None): + def set(self, name: str, value: Optional[str] = None) -> None: if value is None: self.run_config_script('set', name) else: self.run_config_script('set', name, value) - def unset(self, name): + def unset(self, name: str) -> None: self.run_config_script('unset', name) - def batch(self, name): + def batch(self, name: str) -> None: self.run_config_script(name) - def finish(self): + def finish(self) -> None: pass _config_classes['copy'] = ConfigCopy -class ConfigInclude(ConfigMaker): - """ConfigMaker implementation that makes a config script that #includes the base one.""" +class ConfigExplicit(ConfigMaker): + """ConfigMaker implementation that makes a config script with only explicitly set options.""" - def __init__(self, *args): + def __init__(self, *args) -> None: super().__init__(*args) - self.lines = [] + self.lines = [] #type: List[str] - def start(self): - source_path = self.source_file_path - if not os.path.isabs(source_path): - source_path = os.path.join(os.pardir, os.pardir, source_path) - self.lines.append('#ifndef MBEDTLS_CHECK_CONFIG_H') - self.lines.append('#include "{}"'.format(source_path)) + def start(self) -> None: + self.lines.append('/* Generated by mbedtls-prepare-build */') + self.lines.append('#ifndef MBEDTLS_CONFIG_H') + self.lines.append('#define MBEDTLS_CONFIG_H') self.lines.append('') - def set(self, name, value=None): + def set(self, name: str, value: Optional[str] = None) -> None: if value: self.lines.append('#define ' + name + ' ' + value) else: self.lines.append('#define ' + name) - def unset(self, name): + def unset(self, name: str) -> None: self.lines.append('#undef ' + name) - def finish(self): + def finish(self) -> None: + self.lines.append('') + self.lines.append('#endif') + # Overwrite the configuration file unconditionally. We're used + # to that happening. Eventually we should call open_write_safe(), + # but allow a transition period where we don't check for + # "Generated by", so that existing build trees made before + # this class added a "Generated by" comment line can be reused. + with open(self.target_file, 'w') as out: + for line in self.lines: + out.write(line + '\n') +_config_classes['explicit'] = ConfigExplicit + +class ConfigInclude(ConfigExplicit): + """ConfigMaker implementation that makes a config script that #includes the base one.""" + + def start(self) -> None: + source_path = self.source_file_path + if not os.path.isabs(source_path): + source_path = os.path.join(os.pardir, os.pardir, source_path) + self.lines.append('#ifndef MBEDTLS_CHECK_CONFIG_H') + #TODO: in 2.x, MBEDTLS_USER_CONFIG_FILE needs to go here! + self.lines.append('#include "{}"'.format(source_path)) + self.lines.append('') + + def finish(self) -> None: if self.version < 3: self.lines.append('') self.lines.append('#undef MBEDTLS_CHECK_CONFIG_H') @@ -1437,12 +1997,10 @@ class ConfigInclude(ConfigMaker): self.lines.append('#define mbedtls_iso_c_forbids_empty_translation_units mbedtls_iso_c_forbids_empty_translation_units2') self.lines.append('#include "mbedtls/check_config.h"') self.lines.append('#undef mbedtls_iso_c_forbids_empty_translation_units') - self.lines.append('#endif') - with open(self.target_file, 'w') as out: - for line in self.lines: - out.write(line + '\n') + super().finish() _config_classes['include'] = ConfigInclude + class BuildTreeMaker: """Prepare an Mbed TLS/Crypto build tree. @@ -1454,7 +2012,7 @@ class BuildTreeMaker: Typical usage: BuildTreeMaker(options).run() """ - def __init__(self, options): + def __init__(self, options: Options) -> None: self.options = options self.source_path = os.path.abspath(options.source) options.in_tree_build = are_same_existing_files(self.options.source, @@ -1467,24 +2025,25 @@ class BuildTreeMaker: else: options.config_mode = 'copy' self.config = _config_classes[options.config_mode](options) + self.submodules = self.makefile.submodules - def programs_subdirs(self): + def programs_subdirs(self) -> Iterable[str]: """Detect subdirectories for sample programs.""" tops = ([self.options.source] + [os.path.join(self.options.source, submodule) - for submodule in _submodule_names]) + for submodule in self.submodules]) return [os.path.basename(d) for top in tops for d in glob.glob(os.path.join(top, 'programs', '*')) if os.path.isdir(d)] - def make_subdir(self, subdir): + def make_subdir(self, subdir: List[str]) -> None: """Create the given subdirectory of the build tree.""" path = os.path.join(self.options.dir, *subdir) if not os.path.exists(path): os.makedirs(path) - def make_link(self, target, link): + def make_link(self, target: str, link: str) -> None: """Create a symbolic link called link pointing to target. link is a path relative to the build directory. @@ -1495,25 +2054,27 @@ class BuildTreeMaker: if not os.path.lexists(link_path): os.symlink(target, link_path) - def link_to_source(self, sub_link, link): + def link_to_source(self, + sub_link: typing.List[str], + link: typing.List[str]) -> None: """Create a symbolic link in the build tree to sub_link under the source.""" self.make_link(os.path.join(*([os.pardir] * (len(link) - 1) + ['source'] + sub_link)), os.path.join(*link)) - def link_to_source_maybe(self, link): + def link_to_source_maybe(self, link: typing.List[str]) -> None: """Create a symbolic link in the build tree to a target of the same name in the source tree in any submodule. Check the root first, then the submodules in the order given by - _submodule_names. + SUBMODULE_NAMES. """ - for submodule in [''] + _submodule_names: + for submodule in [''] + SUBMODULE_NAMES: sub_link = [submodule] + link if os.path.exists(os.path.join(self.options.source, *sub_link)): self.link_to_source(sub_link, link) - def link_test_suites(self): + def link_test_suites(self) -> None: # This is a hack due to the weird paths produced by the test generator. # The test generator generates a file with #file directives that # point to "suites/xxx.function". Since we run the compiler from the @@ -1526,7 +2087,7 @@ class BuildTreeMaker: # in a submodule. self.make_link('source/tests/suites', 'suites') - def prepare_source(self): + def prepare_source(self) -> None: """Generate extra source files in the source tree. This is necessary in the Mbed TLS development branch since before 3.0: @@ -1549,29 +2110,61 @@ class BuildTreeMaker: subprocess.check_call([make_command, 'generated_files'], cwd=self.source_path) - def run(self): + def run(self) -> None: """Go ahead and prepate the build tree.""" - for subdir in ([['include', 'mbedtls'], - ['library'], - ['tests', 'src', 'drivers']] + - [['programs', d] for d in self.programs_subdirs()]): + for subdir in ( + [ + ['framework'], + ['include', 'mbedtls'], + ['library'], + ['tests', 'include', 'test', 'drivers'], + ['tests', 'libtestdriver1', 'include-testdriver1', 'everest'], + ['tests', 'libtestdriver1', 'include-testdriver1', 'mbedtls'], + ['tests', 'libtestdriver1', 'include-testdriver1', 'p256-m'], + ['tests', 'libtestdriver1', 'include-testdriver1', 'psa'], + ['tests', 'libtestdriver1', 'library'], + ['tests', 'src', 'drivers'], + ['tests', 'src', 'test_helpers'], + ] + + [['programs', d] for d in self.programs_subdirs()] + ): self.make_subdir(subdir) + if os.path.exists(os.path.join(self.options.source, 'tf-psa-crypto')): + for subdir in [ + ['core'], + ['drivers', 'builtin', 'src'], + ['drivers', 'everest', 'library'], + ['drivers', 'everest', 'library', 'kremlib'], + ['drivers', 'everest', 'library', 'legacy'], + ['drivers', 'p256-m', 'p256-m'], + ['tests', 'libtestdriver1', + 'libtestdriver1', 'tf-psa-crypto', 'include', 'psa'], + ]: + self.make_subdir(subdir) source_link = os.path.join(self.options.dir, 'source') if not self.options.in_tree_build and not os.path.exists(source_link): os.symlink(self.source_path, source_link) - for link in [['include', 'psa'], # hack for psa_constant_names.py - ['scripts'], - ['tests', 'configs'], - ['tests', 'compat.sh'], - ['tests', 'data_files'], - ['tests', 'scripts'], - ['tests', 'ssl-opt.sh']]: + for link in [ + ['framework', 'data_files'], + ['include', 'psa'], # hack for psa_constant_names.py + ['programs', 'test', 'zeroize.c'], + ['scripts'], + ['tests', 'configs'], + ['tests', 'compat.sh'], + ['tests', 'data_files'], + ['tests', 'opt-testcases'], + ['tests', 'scripts'], + ['tests', 'ssl-opt.sh'], + ['tf-psa-crypto'], + ]: self.link_to_source_maybe(link) self.link_test_suites() self.prepare_source() self.makefile.generate() self.config.run() +CLANG_WARNING_CFLAGS = '-Werror -Wall -Wextra -Wdocumentation -Wno-documentation-deprecated-sync -std=c99 -D_DEFAULT_SOURCE' + """Named presets. This is a dictionary mapping preset names to their descriptions. The @@ -1580,13 +2173,75 @@ set for this preset. The field _help in a description has a special meaning: it's the documentation of the preset. """ _preset_options = { - '': {}, # empty preset = use defaults + '': argparse.Namespace(), # empty preset = use defaults + 'alt': argparse.Namespace( + _help='Build with ALT implementations', + config_name='full', + config_set=[ + 'MBEDTLS_AES_ALT', + 'MBEDTLS_ARIA_ALT', + 'MBEDTLS_CAMELLIA_ALT', + 'MBEDTLS_CCM_ALT', + 'MBEDTLS_CHACHA20_ALT', + 'MBEDTLS_CHACHAPOLY_ALT', + 'MBEDTLS_CMAC_ALT', + 'MBEDTLS_DES_ALT', + 'MBEDTLS_DHM_ALT', + 'MBEDTLS_ECJPAKE_ALT', + 'MBEDTLS_ECP_ALT', + 'MBEDTLS_GCM_ALT', + #'MBEDTLS_MD2_ALT', + #'MBEDTLS_MD4_ALT', + 'MBEDTLS_MD5_ALT', + 'MBEDTLS_NIST_KW_ALT', + 'MBEDTLS_POLY1305_ALT', + 'MBEDTLS_RIPEMD160_ALT', + 'MBEDTLS_RSA_ALT', + 'MBEDTLS_SHA1_ALT', + 'MBEDTLS_SHA256_ALT', + 'MBEDTLS_SHA512_ALT', + 'MBEDTLS_THREADING_ALT', + 'MBEDTLS_TIMING_ALT', + ], + config_unset=[ + 'MBEDTLS_AESCE_C', + 'MBEDTLS_AESNI_C', + 'MBEDTLS_DEBUG_C', + 'MBEDTLS_PADLOCK_C', + 'MBEDTLS_ECP_RESTARTABLE', + 'MBEDTLS_THREADING_PTHREAD', + 'MBEDTLS_PK_PARSE_EC_EXTENDED', + 'MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_IF_PRESENT', + 'MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY', + 'MBEDTLS_SHA512_USE_A64_CRYPTO_IF_PRESENT', + 'MBEDTLS_SHA512_USE_A64_CRYPTO_ONLY', + ], + default_target='lib', + LIBRARY_EXTRA_CFLAGS='-I $(SOURCE_DIR)/tests/include/alt-dummy' + ), 'asan': argparse.Namespace( _help='Clang with ASan+UBSan, current configuration', config_unset=['MBEDTLS_MEMORY_BUFFER_ALLOC_C'], CC='clang', - CFLAGS='-O', + CFLAGS='-O2', COMMON_FLAGS='-fsanitize=address,undefined -fno-sanitize-recover=all -fno-common -g3', + WARNING_CFLAGS=CLANG_WARNING_CFLAGS, + ), + 'asan-gcc': argparse.Namespace( + _help='GCC with ASan+UBSan, current configuration', + config_unset=['MBEDTLS_MEMORY_BUFFER_ALLOC_C'], + CC='gcc', + CFLAGS='-O2', + COMMON_FLAGS='-fsanitize=address,undefined -fno-sanitize-recover=all -fno-common -g3', + ), + 'cf': argparse.Namespace( + _help='Constant flow with MSan, current configuration', + config_set=['MBEDTLS_TEST_CONSTANT_FLOW_MEMSAN'], + config_unset=['MBEDTLS_AESNI_C'], + CC='clang', + CFLAGS='-O2', + COMMON_FLAGS='-fsanitize=memory -g3', + WARNING_CFLAGS=CLANG_WARNING_CFLAGS, ), 'coverage': argparse.Namespace( _help='Build with coverage instrumentation', @@ -1601,18 +2256,55 @@ _preset_options = { CFLAGS='-O0', COMMON_FLAGS='-g3', ), + 'ecc-heap': argparse.Namespace( + _help='Like scripts/ecc-heap.sh', + config_mode='explicit', + config_set=[ + 'MBEDTLS_PLATFORM_C', + 'MBEDTLS_PLATFORM_MEMORY', + 'MBEDTLS_MEMORY_BUFFER_ALLOC_C', + 'MBEDTLS_MEMORY_DEBUG', + 'MBEDTLS_TIMING_C', + 'MBEDTLS_BIGNUM_C', + 'MBEDTLS_ECP_C', + 'MBEDTLS_ASN1_PARSE_C', + 'MBEDTLS_ASN1_WRITE_C', + 'MBEDTLS_ECDSA_C', + 'MBEDTLS_ECDH_C', + 'MBEDTLS_ECP_DP_SECP192R1_ENABLED', + 'MBEDTLS_ECP_DP_SECP224R1_ENABLED', + 'MBEDTLS_ECP_DP_SECP256R1_ENABLED', + 'MBEDTLS_ECP_DP_SECP384R1_ENABLED', + 'MBEDTLS_ECP_DP_SECP521R1_ENABLED', + 'MBEDTLS_ECP_DP_CURVE25519_ENABLED', + 'MBEDTLS_SHA256_C', #necessary for the ECDSA benchmark + 'MBEDTLS_SHA224_C', #In 3.0, SHA256 requires SHA224 + 'MBEDTLS_ECP_WINDOW_SIZE=4', + 'MBEDTLS_ECP_FIXED_POINT_OPTIM=1', + ], + CFLAGS='-O2', + ), 'full': argparse.Namespace( _help='Full configuration', config_name='full', config_unset=['MBEDTLS_MEMORY_BUFFER_ALLOC_C'], ), 'full-asan': argparse.Namespace( + _help='Full configuration with Clang+ASan+UBSan', + config_name='full', + config_unset=['MBEDTLS_MEMORY_BUFFER_ALLOC_C'], + CC='clang', + CFLAGS='-O2', + COMMON_FLAGS='-fsanitize=address,undefined -fno-sanitize-recover=all -fno-common -g3', + WARNING_CFLAGS=CLANG_WARNING_CFLAGS, + ), + 'full-asan-gcc': argparse.Namespace( _help='Full configuration with GCC+ASan+UBSan', config_name='full', config_unset=['MBEDTLS_MEMORY_BUFFER_ALLOC_C'], CC='gcc', - CFLAGS='-O', - COMMON_FLAGS='-fsanitize=address,undefined -fno-common -g3', + CFLAGS='-O2', + COMMON_FLAGS='-fsanitize=address,undefined -fno-sanitize-recover=all -fno-common -g3', ), 'full-debug': argparse.Namespace( _help='Full configuration, debug build', @@ -1629,9 +2321,23 @@ _preset_options = { CFLAGS='-Os', COMMON_FLAGS='-mthumb', ), + 'iar-thumb': argparse.Namespace( + _help='Baremetal+ configuration built with IAR', + config_name='baremetal', + config_unset=['MBEDTLS_PLATFORM_FPRINTF_ALT', + 'MBEDTLS_PLATFORM_SETBUF_ALT'], + default_target='lib', + CC='iccarm', + CFLAGS='-Ohz --cpu_mode=thumb --cpu=Cortex-M0', + WARNING_CFLAGS='--warnings_are_errors', + ), 'm0plus': argparse.Namespace( _help='Baremetal configuration for Cortex-M0+ target', - config_name='baremetal', + # 'baremetal_size' only exists since shortly before mbedtls-3.2.0 + # and mbedtls-2.28.1. In older versions, use the 'baremetal' + # configuration and unset + # 'MBEDTLS_DEBUG_C,MBEDTLS_SELF_TEST,MBEDTLS_TEST_HOOKS'. + config_name='baremetal_size', default_target='lib', CC='arm-none-eabi-gcc', CFLAGS='-Os', @@ -1649,8 +2355,36 @@ _preset_options = { config_unset=['MBEDTLS_AESNI_C', 'MBEDTLS_MEMORY_BUFFER_ALLOC_C'], CC='clang', - CFLAGS='-O', + CFLAGS='-O2', COMMON_FLAGS='-fsanitize=memory -g3', + WARNING_CFLAGS=CLANG_WARNING_CFLAGS, + ), + 'noplatform': argparse.Namespace( + _help='Full except platform/fsio/net', + config_name='full', + config_unset=[ + 'MBEDTLS_PLATFORM_C', + 'MBEDTLS_NET_C', + 'MBEDTLS_PLATFORM_MEMORY', + 'MBEDTLS_PLATFORM_PRINTF_ALT', + 'MBEDTLS_PLATFORM_FPRINTF_ALT', + 'MBEDTLS_PLATFORM_SNPRINTF_ALT', + 'MBEDTLS_PLATFORM_VSNPRINTF_ALT', + 'MBEDTLS_PLATFORM_TIME_ALT', + 'MBEDTLS_PLATFORM_EXIT_ALT', + 'MBEDTLS_PLATFORM_SETBUF_ALT', + 'MBEDTLS_PLATFORM_NV_SEED_ALT', + 'MBEDTLS_ENTROPY_NV_SEED', + 'MBEDTLS_FS_IO', + 'MBEDTLS_PSA_CRYPTO_SE_C', + 'MBEDTLS_PSA_CRYPTO_STORAGE_C', + 'MBEDTLS_PSA_ITS_FILE_C', + ], + WARNING_CFLAGS='-Werror -Wall -Wextra -std=c99 -D_DEFAULT_SOURCE', + ), + 'opt': argparse.Namespace( + _help='Optimized build', + CFLAGS='-O3', ), 'thumb': argparse.Namespace( _help='Default configuration for arm-linux-gnueabi', @@ -1658,6 +2392,15 @@ _preset_options = { CFLAGS='-Os', COMMON_FLAGS='-mthumb', ), + 'tsan': argparse.Namespace( + _help='Clang with TSan, current configuration + pthread', + config_set=['MBEDTLS_THREADING_C', 'MBEDTLS_THREADING_PTHREAD'], + config_unset=['MBEDTLS_MEMORY_BUFFER_ALLOC_C'], + CC='clang', + CFLAGS='-O2', + COMMON_FLAGS='-fsanitize=thread -fno-sanitize-recover=all -fno-common -g3', + WARNING_CFLAGS=CLANG_WARNING_CFLAGS, + ), 'valgrind': argparse.Namespace( # This is misleading: it doesn't actually run programs through # valgrind when you run e.g. `make check` @@ -1665,7 +2408,7 @@ _preset_options = { config_unset=['MBEDTLS_AESNI_C'], CFLAGS='-g -O3', ), -} +} #type: Dict[str, argparse.Namespace] """Default values for some options. @@ -1684,13 +2427,14 @@ _default_options = { 'source': os.curdir, } -def set_default_option(options, attr, value): +def set_default_option(options: Options, attr: str, + value: typing.Any) -> None: if getattr(options, attr) is None: setattr(options, attr, value) elif isinstance(value, list): setattr(options, attr, value + getattr(options, attr)) -def handle_cross(options): +def handle_cross(options: Options) -> None: """Update settings to handle --cross.""" if options.cross is None: return @@ -1700,7 +2444,7 @@ def handle_cross(options): set_default_option(options, 'CC', options.cross + '-gcc') set_default_option(options, 'dir', 'build-' + options.cross) -def set_default_options(options): +def set_default_options(options: Options) -> None: """Apply the preset if any and set default for remaining options. We set defaults via this function rather than via the `default` @@ -1729,7 +2473,42 @@ def set_default_options(options): for envopt in _environment_options: set_default_option(options, envopt.attr, envopt.default) -def preset_help(): +_USER_LIST_SEPARATOR_RE = re.compile(r'[\s,]+') +def split_user_list(user_list: Iterable[str]) -> Iterable[str]: + """In a list of strings, split comma- or whitespace-separated pieces of each element.""" + for element in user_list: + yield from _USER_LIST_SEPARATOR_RE.split(element) + +def normalize_psa_mechanism(name: str, prefix: str) -> str: + """Normalize a PSA mechanism name. + + Ensure that each word starts with the given prefix: PSA_ or PSA_WANT_ + or MBEDTLS_PSA_BUILTIN_ or MBEDTLS_PSA_ACCEL_. Any such prefix is first + removed from the word. + """ + name = re.sub(r'\A(?:MBEDTLS_PSA_BUILTIN_|MBEDTLS_PSA_ACCEL|PSA_|PSA_WANT_)', + r'', name) + return prefix + name + +def normalize_psa_mechanism_list(user_list: List[str], prefix: str) -> List[str]: + """Normalize a list of PSA mechanisms. + + 1. Separate into comma/whitespace-separated words. + 2. Ensure that each word starts with the given prefix: PSA_ or PSA_WANT_ + or MBEDTLS_PSA_BUILTIN_ or MBEDTLS_PSA_ACCEL_. Any such prefix is first + removed from the word. + """ + return [normalize_psa_mechanism(mechanism, prefix) + for mechanism in split_user_list(user_list) + if mechanism] + +def normalize_options(options) -> None: + """Normalize some options that can be passed in easier-to-type ways.""" + options.accel_list = normalize_psa_mechanism_list(options.accel_list, '') + options.libtestdriver1_extra_list = \ + normalize_psa_mechanism_list(options.libtestdriver1_extra_list, '') + +def preset_help() -> str: """Return a documentation string for the presets.""" return '\n'.join(['Presets:'] + ['{}: {}'.format(name, _preset_options[name]._help) @@ -1737,7 +2516,7 @@ def preset_help(): if hasattr(_preset_options[name], '_help')] + ['']) -def arg_type_bool(arg): +def arg_type_bool(arg: typing.Union[bool, str]) -> bool: """Boolean argument type for argparse.add_argument.""" if not isinstance(arg, str): return arg @@ -1749,7 +2528,7 @@ def arg_type_bool(arg): else: raise argparse.ArgumentTypeError('invalid boolean value: ' + repr(arg)) -def main(): +def main() -> None: """Process the command line and prepare a build tree accordingly.""" parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, description=__doc__, @@ -1758,6 +2537,11 @@ def main(): parser.add_argument(envopt.option, dest=envopt.attr, help='{} ({})'.format(envopt.help, envopt.var)) + parser.add_argument('--accel-list', + action='append', default=[], + help='Algorithms to accelerate through libtestdriver1.' + ' This is loc_accel_list in all.sh.' + ' No libtestdriver1 if empty.') parser.add_argument('--assembly-extension', help='File extension for assembly files') parser.add_argument('--config-file', @@ -1789,6 +2573,11 @@ def main(): help='Whether to use makefile variable for file extensions') parser.add_argument('--library-extension', help='File extension for static libraries') + parser.add_argument('--libtestdriver1-extra-list', + action='append', default=[], + help='Extra algorithms to enable in libtestdriver1.' + ' This is loc_extra_list in all.sh.' + ' Ignored if --accel-list is empty.') parser.add_argument('--object-extension', help='File extension for object files') parser.add_argument('--preset', '-p', @@ -1805,6 +2594,7 @@ def main(): action='append', default=[], help='Extra variable to define in the makefile') options = parser.parse_args() + normalize_options(options) set_default_options(options) builder = BuildTreeMaker(options) builder.run() diff --git a/tools/bin/mbedtls-run-tests b/tools/bin/mbedtls-run-tests index e73f8cf..9e6fc96 100755 --- a/tools/bin/mbedtls-run-tests +++ b/tools/bin/mbedtls-run-tests @@ -12,12 +12,12 @@ import os import re import subprocess import tempfile - +import typing class TestCaseFilter: """Test case filter.""" - def __init__(self, options): + def __init__(self, options: argparse.Namespace) -> None: """Set up a test case filter. See `main` for what options are valid. @@ -44,7 +44,7 @@ class TestCaseFilter: not re.match(self.exclude, description)) -def extract_description(stanza): +def extract_description(stanza: str) -> typing.Optional[str]: """Extract the description from a .data stanza.""" m = re.match(r'(?:\n|#[^\n]*\n)*([^#\n][^\n]*)\n', stanza + '\n') if m: @@ -52,7 +52,9 @@ def extract_description(stanza): else: return None -def filter_test_cases(all_datax, tcf, temp_datax): +def filter_test_cases(all_datax: str, + tcf: TestCaseFilter, + temp_datax: typing.IO[bytes]) -> None: """Filter test cases. Filter test cases from datax_file based on their description according @@ -71,7 +73,12 @@ def filter_test_cases(all_datax, tcf, temp_datax): temp_datax.write(line) temp_datax.flush() -def run_exe(keep_temp, precommand, exe, extra_options, all_datax, tcf): +def run_exe(keep_temp: bool, + precommand: typing.List[str], + exe: str, + extra_options: typing.List[str], + all_datax: str, + tcf: TestCaseFilter) -> int: """Run one Mbed TLS test suites based on the specified options. Return the subprocess's exit status (should be 0 for success, 1 for @@ -95,7 +102,7 @@ def run_exe(keep_temp, precommand, exe, extra_options, all_datax, tcf): outcome = subprocess.run(cmd, cwd=directory) return outcome.returncode -def find_data_file(exe): +def find_data_file(exe: str) -> str: """Return the .datax file for the specified test suite executable.""" directory = os.path.dirname(exe) basename = os.path.basename(exe) @@ -109,7 +116,7 @@ def find_data_file(exe): else: raise Exception('.datax file not found for ' + exe) -def run(options): +def run(options: argparse.Namespace) -> int: """Run Mbed TLS test suites based on the specified options. See `main` for what options are valid. @@ -131,7 +138,7 @@ def run(options): global_status = max(global_status, 120) return global_status -def main(): +def main() -> None: """Process the command line and run Mbed TLS tests.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--command', '-c', diff --git a/tools/zsh/_config.pl b/tools/zsh/_config.pl index 08ba7bc..e9bc7f0 100644 --- a/tools/zsh/_config.pl +++ b/tools/zsh/_config.pl @@ -1,9 +1,12 @@ #compdef config.pl config.py -## Completion for scripts/config.pl and scripts/config.pl in Mbed TLS. +## Completion for scripts/config.pl and scripts/config.py in Mbed TLS. _config_pl_symbols () { local -a identifiers - identifiers=("${(@f)$(_call_program _config_pl_symbols sed -n \''s!^/*\**#define \(MBEDTLS_[0-9A-Z_a-z][0-9A-Z_a-z]*\).*!\1!p'\' \$config_h)}") + identifiers=("${(@f)$(_call_program _config_pl_symbols 'sed -n \ + -e '\''s!^/*\**#define \(MBEDTLS_[0-9A-Z_a-z][0-9A-Z_a-z]*\).*!\1!p'\'' \ + -e '\''s!^/*\**#define \(PSA_[0-9A-Z_a-z][0-9A-Z_a-z]*\).*!\1!p'\'' \ + $config_h')}") _describe -t symbols 'config.h symbols' identifiers } @@ -26,6 +29,11 @@ {'-o','--force'}'[define symbol even if not present]' \ '1:config.pl command:->command' \ '*::config.pl commands:->param' + if (($+opt_args[--file])); then + config_h=$opt_args[--file] + elif (($+opt_args[-f])); then + config_h=$opt_args[-f] + fi case $state in (command) _describe -t commands 'config.pl command' commands_with_descriptions;; diff --git a/tools/zsh/_mbedtls_programs b/tools/zsh/_mbedtls_programs new file mode 100644 index 0000000..6a40ace --- /dev/null +++ b/tools/zsh/_mbedtls_programs @@ -0,0 +1,30 @@ +#compdef dh_genprime gen_key key_app key_app_writer ssl_client2 ssl_mail_client ssl_server2 pem2der cert_app cert_req cert_write crl_app req_app +## Completion for Mbed TLS SSL sample and test programs. + +_ssl_client2 () { + local param + local -a params values + params=("${(@)${(@)${(@M)${(@f)$(_call_program help $words[1] help)}:# #[a-z][0-9A-Z_a-z]#=*}%%=*}## ##}") + if [[ $PREFIX$SUFFIX == *=* ]]; then + IPREFIX="${PREFIX%%\=*}=" + PREFIX="${PREFIX#*=}" + param=${${IPREFIX%=}##[!0-9A-Z_a-z]##} + case $param in + auth_mode) + _values $param none optional required;; + force_ciphersuite) + values=("${(@M)${=$(_call_program help_ciphersuites $words[1] help_ciphersuites)}:#TLS-*}") + _values $param $values;; + *path*|output_file) + _files -/;; + *file*|*_crt|*_key) + _files;; + *version) + _values $param ssl3 tls10 tls11 tls12 tls13 dtls10 dtls12;; + esac + else + _values -s= parameter $params + fi +} + +_ssl_client2 "$@"