Merge remote-tracking branch 'origin/development' into development-restricted

* origin/development: (33 commits)
  Test with MBEDTLS_ECP_RESTARTABLE
  Allow TODO in code
  Use the docstring in the command line help
  Split _abi_compliance_command into smaller functions
  Record the commits that were compared
  Document how to build the typical argument for -s
  Allow running /somewhere/else/path/to/abi_check.py
  Warn if VLAs are used
  Remove redundant compiler flag
  Consistently spell -Wextra
  Update Mbed Crypto to contain mbed-crypto#152
  Improve compatibility with firewalled networks
  Dockerfile: apt -> apt-get
  Change Docker container to bionic
  Clean up file prologue comments
  Add docker-based test scripts
  ChangeLog: Add ChangeLog entry for #2681
  Allow declarations after statements
  CMake: Add a subdirectory build regression test
  README: Enable builds as a CMake subproject
  ...

tests/scripts/all-in-docker.sh

@@ -0,0 +1,37 @@
+#!/bin/bash -eu
+
+# all-in-docker.sh
+#
+# Purpose
+# -------
+# This runs all.sh (except for armcc) in a Docker container.
+#
+# Notes for users
+# ---------------
+# See docker_env.sh for prerequisites and other information.
+#
+# See also all.sh for notes about invocation of that script.
+
+# Copyright (C) 2006-2019, Arm Limited (or its affiliates), All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This file is part of Mbed TLS (https://tls.mbed.org)
+
+source tests/scripts/docker_env.sh
+
+# Run tests that are possible with openly available compilers
+run_in_docker tests/scripts/all.sh \
+    --no-armcc \
+    $@

tests/scripts/all.sh

@@ -242,6 +242,11 @@ cleanup()
     git checkout -- Makefile library/Makefile programs/Makefile tests/Makefile
     cd ..
 
+    # Remove any artifacts from the component_test_cmake_as_subdirectory test.
+    rm -rf programs/test/cmake_subproject/build
+    rm -f programs/test/cmake_subproject/Makefile
+    rm -f programs/test/cmake_subproject/cmake_subproject
+
     if [ -f "$CONFIG_BAK" ]; then
         mv "$CONFIG_BAK" "$CONFIG_H"
     fi
@@ -836,7 +841,7 @@ component_test_no_use_psa_crypto_full_cmake_asan() {
     msg "build: cmake, full config + MBEDTLS_USE_PSA_CRYPTO, ASan"
     scripts/config.pl full
     scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE # too slow for tests
-    scripts/config.pl unset MBEDTLS_ECP_RESTARTABLE  # restartable ECC not supported through PSA
+    scripts/config.pl set MBEDTLS_ECP_RESTARTABLE  # not using PSA, so enable restartable ECC
     scripts/config.pl set MBEDTLS_PSA_CRYPTO_C
     scripts/config.pl unset MBEDTLS_USE_PSA_CRYPTO
     scripts/config.pl unset MBEDTLS_PSA_ITS_FILE_C
@@ -1236,6 +1241,19 @@ component_test_cmake_out_of_source () {
     unset MBEDTLS_ROOT_DIR
 }
 
+component_test_cmake_as_subdirectory () {
+    msg "build: cmake 'as-subdirectory' build"
+    MBEDTLS_ROOT_DIR="$PWD"
+
+    cd programs/test/cmake_subproject
+    cmake .
+    make
+    if_build_succeeded ./cmake_subproject
+
+    cd "$MBEDTLS_ROOT_DIR"
+    unset MBEDTLS_ROOT_DIR
+}
+
 component_test_zeroize () {
     # Test that the function mbedtls_platform_zeroize() is not optimized away by
     # different combinations of compilers and optimization flags by using an

tests/scripts/basic-in-docker.sh

@@ -0,0 +1,50 @@
+#!/bin/bash -eu
+
+# basic-in-docker.sh
+#
+# Purpose
+# -------
+# This runs a rough equivalent of the travis.yml in a Docker container.
+# The tests are run for both clang and gcc.
+#
+# Notes for users
+# ---------------
+# See docker_env.sh for prerequisites and other information.
+
+# Copyright (C) 2006-2019, Arm Limited (or its affiliates), All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This file is part of Mbed TLS (https://tls.mbed.org)
+
+source tests/scripts/docker_env.sh
+
+run_in_docker tests/scripts/recursion.pl library/*.c
+run_in_docker tests/scripts/check-generated-files.sh
+run_in_docker tests/scripts/check-doxy-blocks.pl
+run_in_docker tests/scripts/check-names.sh
+run_in_docker tests/scripts/check-files.py
+run_in_docker tests/scripts/doxygen.sh
+
+for compiler in clang gcc; do
+    run_in_docker -e CC=${compiler} cmake -D CMAKE_BUILD_TYPE:String="Check" .
+    run_in_docker -e CC=${compiler} make
+    run_in_docker -e CC=${compiler} make test
+    run_in_docker programs/test/selftest
+    run_in_docker -e OSSL_NO_DTLS=1 tests/compat.sh
+    run_in_docker tests/ssl-opt.sh -e '\(DTLS\|SCSV\).*openssl'
+    run_in_docker tests/scripts/test-ref-configs.pl
+    run_in_docker tests/scripts/curves.pl
+    run_in_docker tests/scripts/key-exchanges.pl
+done

tests/scripts/check-files.py

@@ -1,14 +1,12 @@
 #!/usr/bin/env python3
+
+# This file is part of Mbed TLS (https://tls.mbed.org)
+# Copyright (c) 2018, Arm Limited, All Rights Reserved
+
 """
-This file is part of Mbed TLS (https://tls.mbed.org)
-
-Copyright (c) 2018, Arm Limited, All Rights Reserved
-
-Purpose
-
 This script checks the current state of the source code for minor issues,
 including incorrect file permissions, presence of tabs, non-Unix line endings,
-trailing whitespace, presence of UTF-8 BOM, and TODO comments.
+trailing whitespace, and presence of UTF-8 BOM.
 Note: requires python 3, must be run from Mbed TLS root.
 """
 
@@ -170,19 +168,6 @@ class MergeArtifactIssueTracker(LineIssueTracker):
             return True
         return False
 
-class TodoIssueTracker(LineIssueTracker):
-    """Track lines containing ``TODO``."""
-
-    heading = "TODO present:"
-    files_exemptions = frozenset([
-        os.path.basename(__file__),
-        "benchmark.c",
-        "pull_request_template.md",
-    ])
-
-    def issue_with_line(self, line, _filepath):
-        return b"todo" in line.lower()
-
 
 class IntegrityChecker(object):
     """Sanity-check files under the current directory."""
@@ -211,7 +196,6 @@ class IntegrityChecker(object):
             TrailingWhitespaceIssueTracker(),
             TabIssueTracker(),
             MergeArtifactIssueTracker(),
-            TodoIssueTracker(),
         ]
 
     @staticmethod
@@ -257,15 +241,7 @@ class IntegrityChecker(object):
 
 
 def run_main():
-    parser = argparse.ArgumentParser(
-        description=(
-            "This script checks the current state of the source code for "
-            "minor issues, including incorrect file permissions, "
-            "presence of tabs, non-Unix line endings, trailing whitespace, "
-            "presence of UTF-8 BOM, and TODO comments. "
-            "Note: requires python 3, must be run from Mbed TLS root."
-        )
-    )
+    parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument(
         "-l", "--log_file", type=str, help="path to optional output log",
     )

tests/scripts/docker_env.sh

@@ -0,0 +1,93 @@
+#!/bin/bash -eu
+
+# docker_env.sh
+#
+# Purpose
+# -------
+#
+# This is a helper script to enable running tests under a Docker container,
+# thus making it easier to get set up as well as isolating test dependencies
+# (which include legacy/insecure configurations of openssl and gnutls).
+#
+# Notes for users
+# ---------------
+# This script expects a Linux x86_64 system with a recent version of Docker
+# installed and available for use, as well as http/https access. If a proxy
+# server must be used, invoke this script with the usual environment variables
+# (http_proxy and https_proxy) set appropriately. If an alternate Docker
+# registry is needed, specify MBEDTLS_DOCKER_REGISTRY to point at the
+# host name.
+#
+#
+# Running this script directly will check for Docker availability and set up
+# the Docker image.
+
+# Copyright (C) 2006-2019, Arm Limited (or its affiliates), All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This file is part of Mbed TLS (https://tls.mbed.org)
+
+
+# default values, can be overridden by the environment
+: ${MBEDTLS_DOCKER_GUEST:=bionic}
+
+
+DOCKER_IMAGE_TAG="armmbed/mbedtls-test:${MBEDTLS_DOCKER_GUEST}"
+
+# Make sure docker is available
+if ! which docker > /dev/null; then
+    echo "Docker is required but doesn't seem to be installed. See https://www.docker.com/ to get started"
+    exit 1
+fi
+
+# Figure out if we need to 'sudo docker'
+if groups | grep docker > /dev/null; then
+    DOCKER="docker"
+else
+    echo "Using sudo to invoke docker since you're not a member of the docker group..."
+    DOCKER="sudo docker"
+fi
+
+# Build the Docker image
+echo "Getting docker image up to date (this may take a few minutes)..."
+${DOCKER} image build \
+    -t ${DOCKER_IMAGE_TAG} \
+    --cache-from=${DOCKER_IMAGE_TAG} \
+    --build-arg MAKEFLAGS_PARALLEL="-j $(nproc)" \
+    --network host \
+    ${http_proxy+--build-arg http_proxy=${http_proxy}} \
+    ${https_proxy+--build-arg https_proxy=${https_proxy}} \
+    ${MBEDTLS_DOCKER_REGISTRY+--build-arg MY_REGISTRY="${MBEDTLS_DOCKER_REGISTRY}/"} \
+    tests/docker/${MBEDTLS_DOCKER_GUEST}
+
+run_in_docker()
+{
+    ENV_ARGS=""
+    while [ "$1" == "-e" ]; do
+        ENV_ARGS="${ENV_ARGS} $1 $2"
+        shift 2
+    done
+
+    ${DOCKER} container run -it --rm \
+        --cap-add SYS_PTRACE \
+        --user "$(id -u):$(id -g)" \
+        --volume $PWD:$PWD \
+        --workdir $PWD \
+        -e MAKEFLAGS \
+        -e PYLINTHOME=/tmp/.pylintd \
+        ${ENV_ARGS} \
+        ${DOCKER_IMAGE_TAG} \
+        $@
+}