Merge remote-tracking branch 'origin/development' into misc-code-size

scripts/mbedtls_dev/crypto_data_tests.py

@@ -0,0 +1,123 @@
+"""Generate test data for cryptographic mechanisms.
+
+This module is a work in progress, only implementing a few cases for now.
+"""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import hashlib
+from typing import Callable, Dict, Iterator, List, Optional #pylint: disable=unused-import
+
+from . import crypto_knowledge
+from . import psa_information
+from . import test_case
+
+
+def psa_low_level_dependencies(*expressions: str) -> List[str]:
+    """Infer dependencies of a PSA low-level test case by looking for PSA_xxx symbols.
+
+    This function generates MBEDTLS_PSA_BUILTIN_xxx symbols.
+    """
+    high_level = psa_information.automatic_dependencies(*expressions)
+    for dep in high_level:
+        assert dep.startswith('PSA_WANT_')
+    return ['MBEDTLS_PSA_BUILTIN_' + dep[9:] for dep in high_level]
+
+
+class HashPSALowLevel:
+    """Generate test cases for the PSA low-level hash interface."""
+
+    def __init__(self, info: psa_information.Information) -> None:
+        self.info = info
+        base_algorithms = sorted(info.constructors.algorithms)
+        all_algorithms = \
+            [crypto_knowledge.Algorithm(expr)
+             for expr in info.constructors.generate_expressions(base_algorithms)]
+        self.algorithms = \
+            [alg
+             for alg in all_algorithms
+             if (not alg.is_wildcard and
+                 alg.can_do(crypto_knowledge.AlgorithmCategory.HASH))]
+
+    # CALCULATE[alg] = function to return the hash of its argument in hex
+    # TO-DO: implement the None entries with a third-party library, because
+    # hashlib might not have everything, depending on the Python version and
+    # the underlying OpenSSL. On Ubuntu 16.04, truncated sha512 and sha3/shake
+    # are not available. On Ubuntu 22.04, md2, md4 and ripemd160 are not
+    # available.
+    CALCULATE = {
+        'PSA_ALG_MD5': lambda data: hashlib.md5(data).hexdigest(),
+        'PSA_ALG_RIPEMD160': None, #lambda data: hashlib.new('ripdemd160').hexdigest()
+        'PSA_ALG_SHA_1': lambda data: hashlib.sha1(data).hexdigest(),
+        'PSA_ALG_SHA_224': lambda data: hashlib.sha224(data).hexdigest(),
+        'PSA_ALG_SHA_256': lambda data: hashlib.sha256(data).hexdigest(),
+        'PSA_ALG_SHA_384': lambda data: hashlib.sha384(data).hexdigest(),
+        'PSA_ALG_SHA_512': lambda data: hashlib.sha512(data).hexdigest(),
+        'PSA_ALG_SHA_512_224': None, #lambda data: hashlib.new('sha512_224').hexdigest()
+        'PSA_ALG_SHA_512_256': None, #lambda data: hashlib.new('sha512_256').hexdigest()
+        'PSA_ALG_SHA3_224': None, #lambda data: hashlib.sha3_224(data).hexdigest(),
+        'PSA_ALG_SHA3_256': None, #lambda data: hashlib.sha3_256(data).hexdigest(),
+        'PSA_ALG_SHA3_384': None, #lambda data: hashlib.sha3_384(data).hexdigest(),
+        'PSA_ALG_SHA3_512': None, #lambda data: hashlib.sha3_512(data).hexdigest(),
+        'PSA_ALG_SHAKE256_512': None, #lambda data: hashlib.shake_256(data).hexdigest(64),
+    } #type: Dict[str, Optional[Callable[[bytes], str]]]
+
+    @staticmethod
+    def one_test_case(alg: crypto_knowledge.Algorithm,
+                      function: str, note: str,
+                      arguments: List[str]) -> test_case.TestCase:
+        """Construct one test case involving a hash."""
+        tc = test_case.TestCase()
+        tc.set_description('{}{} {}'
+                           .format(function,
+                                   ' ' + note if note else '',
+                                   alg.short_expression()))
+        tc.set_dependencies(psa_low_level_dependencies(alg.expression))
+        tc.set_function(function)
+        tc.set_arguments([alg.expression] +
+                         ['"{}"'.format(arg) for arg in arguments])
+        return tc
+
+    def test_cases_for_hash(self,
+                            alg: crypto_knowledge.Algorithm
+                            ) -> Iterator[test_case.TestCase]:
+        """Enumerate all test cases for one hash algorithm."""
+        calc = self.CALCULATE[alg.expression]
+        if calc is None:
+            return # not implemented yet
+
+        short = b'abc'
+        hash_short = calc(short)
+        long = (b'Hello, world. Here are 16 unprintable bytes: ['
+                b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a'
+                b'\x80\x81\x82\x83\xfe\xff]. '
+                b' This message was brought to you by a natural intelligence. '
+                b' If you can read this, good luck with your debugging!')
+        hash_long = calc(long)
+
+        yield self.one_test_case(alg, 'hash_empty', '', [calc(b'')])
+        yield self.one_test_case(alg, 'hash_valid_one_shot', '',
+                                 [short.hex(), hash_short])
+        for n in [0, 1, 64, len(long) - 1, len(long)]:
+            yield self.one_test_case(alg, 'hash_valid_multipart',
+                                     '{} + {}'.format(n, len(long) - n),
+                                     [long[:n].hex(), calc(long[:n]),
+                                      long[n:].hex(), hash_long])
+
+    def all_test_cases(self) -> Iterator[test_case.TestCase]:
+        """Enumerate all test cases for all hash algorithms."""
+        for alg in self.algorithms:
+            yield from self.test_cases_for_hash(alg)

scripts/mbedtls_dev/logging_util.py

@@ -0,0 +1,57 @@
+"""Auxiliary functions used for logging module.
+"""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import sys
+
+def configure_logger(
+        logger: logging.Logger,
+        log_format="[%(levelname)s]: %(message)s",
+        split_level=logging.WARNING
+    ) -> None:
+    """
+    Configure the logging.Logger instance so that:
+        - Format is set to any log_format.
+            Default: "[%(levelname)s]: %(message)s"
+        - loglevel >= split_level are printed to stderr.
+        - loglevel <  split_level are printed to stdout.
+            Default: logging.WARNING
+    """
+    class MaxLevelFilter(logging.Filter):
+        # pylint: disable=too-few-public-methods
+        def __init__(self, max_level, name=''):
+            super().__init__(name)
+            self.max_level = max_level
+
+        def filter(self, record: logging.LogRecord) -> bool:
+            return record.levelno <= self.max_level
+
+    log_formatter = logging.Formatter(log_format)
+
+    # set loglevel >= split_level to be printed to stderr
+    stderr_hdlr = logging.StreamHandler(sys.stderr)
+    stderr_hdlr.setLevel(split_level)
+    stderr_hdlr.setFormatter(log_formatter)
+
+    # set loglevel < split_level to be printed to stdout
+    stdout_hdlr = logging.StreamHandler(sys.stdout)
+    stdout_hdlr.addFilter(MaxLevelFilter(split_level - 1))
+    stdout_hdlr.setFormatter(log_formatter)
+
+    logger.addHandler(stderr_hdlr)
+    logger.addHandler(stdout_hdlr)

scripts/mbedtls_dev/psa_information.py

@@ -0,0 +1,162 @@
+"""Collect information about PSA cryptographic mechanisms.
+"""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+from typing import Dict, FrozenSet, List, Optional
+
+from . import macro_collector
+
+
+class Information:
+    """Gather information about PSA constructors."""
+
+    def __init__(self) -> None:
+        self.constructors = self.read_psa_interface()
+
+    @staticmethod
+    def remove_unwanted_macros(
+            constructors: macro_collector.PSAMacroEnumerator
+    ) -> None:
+        # Mbed TLS does not support finite-field DSA.
+        # Don't attempt to generate any related test case.
+        constructors.key_types.discard('PSA_KEY_TYPE_DSA_KEY_PAIR')
+        constructors.key_types.discard('PSA_KEY_TYPE_DSA_PUBLIC_KEY')
+
+    def read_psa_interface(self) -> macro_collector.PSAMacroEnumerator:
+        """Return the list of known key types, algorithms, etc."""
+        constructors = macro_collector.InputsForTest()
+        header_file_names = ['include/psa/crypto_values.h',
+                             'include/psa/crypto_extra.h']
+        test_suites = ['tests/suites/test_suite_psa_crypto_metadata.data']
+        for header_file_name in header_file_names:
+            constructors.parse_header(header_file_name)
+        for test_cases in test_suites:
+            constructors.parse_test_cases(test_cases)
+        self.remove_unwanted_macros(constructors)
+        constructors.gather_arguments()
+        return constructors
+
+
+def psa_want_symbol(name: str) -> str:
+    """Return the PSA_WANT_xxx symbol associated with a PSA crypto feature."""
+    if name.startswith('PSA_'):
+        return name[:4] + 'WANT_' + name[4:]
+    else:
+        raise ValueError('Unable to determine the PSA_WANT_ symbol for ' + name)
+
+def finish_family_dependency(dep: str, bits: int) -> str:
+    """Finish dep if it's a family dependency symbol prefix.
+
+    A family dependency symbol prefix is a PSA_WANT_ symbol that needs to be
+    qualified by the key size. If dep is such a symbol, finish it by adjusting
+    the prefix and appending the key size. Other symbols are left unchanged.
+    """
+    return re.sub(r'_FAMILY_(.*)', r'_\1_' + str(bits), dep)
+
+def finish_family_dependencies(dependencies: List[str], bits: int) -> List[str]:
+    """Finish any family dependency symbol prefixes.
+
+    Apply `finish_family_dependency` to each element of `dependencies`.
+    """
+    return [finish_family_dependency(dep, bits) for dep in dependencies]
+
+SYMBOLS_WITHOUT_DEPENDENCY = frozenset([
+    'PSA_ALG_AEAD_WITH_AT_LEAST_THIS_LENGTH_TAG', # modifier, only in policies
+    'PSA_ALG_AEAD_WITH_SHORTENED_TAG', # modifier
+    'PSA_ALG_ANY_HASH', # only in policies
+    'PSA_ALG_AT_LEAST_THIS_LENGTH_MAC', # modifier, only in policies
+    'PSA_ALG_KEY_AGREEMENT', # chaining
+    'PSA_ALG_TRUNCATED_MAC', # modifier
+])
+def automatic_dependencies(*expressions: str) -> List[str]:
+    """Infer dependencies of a test case by looking for PSA_xxx symbols.
+
+    The arguments are strings which should be C expressions. Do not use
+    string literals or comments as this function is not smart enough to
+    skip them.
+    """
+    used = set()
+    for expr in expressions:
+        used.update(re.findall(r'PSA_(?:ALG|ECC_FAMILY|KEY_TYPE)_\w+', expr))
+    used.difference_update(SYMBOLS_WITHOUT_DEPENDENCY)
+    return sorted(psa_want_symbol(name) for name in used)
+
+# Define set of regular expressions and dependencies to optionally append
+# extra dependencies for test case.
+AES_128BIT_ONLY_DEP_REGEX = r'AES\s(192|256)'
+AES_128BIT_ONLY_DEP = ["!MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH"]
+
+DEPENDENCY_FROM_KEY = {
+    AES_128BIT_ONLY_DEP_REGEX: AES_128BIT_ONLY_DEP
+}#type: Dict[str, List[str]]
+def generate_key_dependencies(description: str) -> List[str]:
+    """Return additional dependencies based on pairs of REGEX and dependencies.
+    """
+    deps = []
+    for regex, dep in DEPENDENCY_FROM_KEY.items():
+        if re.search(regex, description):
+            deps += dep
+
+    return deps
+
+# A temporary hack: at the time of writing, not all dependency symbols
+# are implemented yet. Skip test cases for which the dependency symbols are
+# not available. Once all dependency symbols are available, this hack must
+# be removed so that a bug in the dependency symbols properly leads to a test
+# failure.
+def read_implemented_dependencies(filename: str) -> FrozenSet[str]:
+    return frozenset(symbol
+                     for line in open(filename)
+                     for symbol in re.findall(r'\bPSA_WANT_\w+\b', line))
+_implemented_dependencies = None #type: Optional[FrozenSet[str]] #pylint: disable=invalid-name
+def hack_dependencies_not_implemented(dependencies: List[str]) -> None:
+    global _implemented_dependencies #pylint: disable=global-statement,invalid-name
+    if _implemented_dependencies is None:
+        _implemented_dependencies = \
+            read_implemented_dependencies('include/psa/crypto_config.h')
+    if not all((dep.lstrip('!') in _implemented_dependencies or
+                not dep.lstrip('!').startswith('PSA_WANT'))
+               for dep in dependencies):
+        dependencies.append('DEPENDENCY_NOT_IMPLEMENTED_YET')
+
+def tweak_key_pair_dependency(dep: str, usage: str):
+    """
+    This helper function add the proper suffix to PSA_WANT_KEY_TYPE_xxx_KEY_PAIR
+    symbols according to the required usage.
+    """
+    ret_list = list()
+    if dep.endswith('KEY_PAIR'):
+        if usage == "BASIC":
+            # BASIC automatically includes IMPORT and EXPORT for test purposes (see
+            # config_psa.h).
+            ret_list.append(re.sub(r'KEY_PAIR', r'KEY_PAIR_BASIC', dep))
+            ret_list.append(re.sub(r'KEY_PAIR', r'KEY_PAIR_IMPORT', dep))
+            ret_list.append(re.sub(r'KEY_PAIR', r'KEY_PAIR_EXPORT', dep))
+        elif usage == "GENERATE":
+            ret_list.append(re.sub(r'KEY_PAIR', r'KEY_PAIR_GENERATE', dep))
+    else:
+        # No replacement to do in this case
+        ret_list.append(dep)
+    return ret_list
+
+def fix_key_pair_dependencies(dep_list: List[str], usage: str):
+    new_list = [new_deps
+                for dep in dep_list
+                for new_deps in tweak_key_pair_dependency(dep, usage)]
+
+    return new_list