From 0b62b7a21f026581735f9bd5bf3c0384426c7c67 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 8 Sep 2023 16:19:13 +0200 Subject: [PATCH 1/6] Allow turning off re-generation of files with make In make builds, when GEN_FILES is false (empty), don't try to re-generate configuration-independent source files, regardless of whether they seem out of date. This is useful, for example, if you have a source tree where `make generated_files` has already run and file timestamps reflect the time the files were copied or extracted, and you are now in an environment that lacks some of the necessary tools to re-generate the files. Signed-off-by: Gilles Peskine --- Makefile | 29 +++++++++++++++++++++++++---- library/Makefile | 28 ++++++++++++++++++---------- programs/Makefile | 24 +++++++++++++++++------- tests/Makefile | 14 +++++++++++--- 4 files changed, 71 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 1f36a06c9..b324b9a35 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,27 @@ generated_files: programs/generated_files generated_files: tests/generated_files generated_files: visualc_files +# Set GEN_FILES to the empty string to disable dependencies on generated +# source files. Then `make generated_files` will only build files that +# are missing, it will not rebuilt files that are present but out of date. +# This is useful, for example, if you have a source tree where +# `make generated_files` has already run and file timestamps reflect the +# time the files were copied or extracted, and you are now in an environment +# that lacks some of the necessary tools to re-generate the files. +# If $(GEN_FILES) is non-empty, the generated source files' dependencies +# are treated ordinarily, based on file timestamps. +GEN_FILES ?= yes + +# In dependencies where the target is a configuration-independent generated +# file, use `TARGET: $(gen_file_dep) DEPENDENCY1 DEPENDENCY2 ...` +# rather than directly `TARGET: DEPENDENCY1 DEPENDENCY2 ...`. This +# enables the re-generation to be turned off when GEN_FILES is disabled. +ifdef GEN_FILES +gen_file_dep = +else +gen_file_dep = | +endif + .PHONY: visualc_files VISUALC_FILES = visualc/VS2013/mbedTLS.sln visualc/VS2013/mbedTLS.vcxproj # TODO: $(app).vcxproj for each $(app) in programs/ @@ -45,10 +66,10 @@ visualc_files: $(VISUALC_FILES) # present before it runs. It doesn't matter if the files aren't up-to-date, # they just need to be present. $(VISUALC_FILES): | library/generated_files -$(VISUALC_FILES): scripts/generate_visualc_files.pl -$(VISUALC_FILES): scripts/data_files/vs2013-app-template.vcxproj -$(VISUALC_FILES): scripts/data_files/vs2013-main-template.vcxproj -$(VISUALC_FILES): scripts/data_files/vs2013-sln-template.sln +$(VISUALC_FILES): $(gen_file_dep) scripts/generate_visualc_files.pl +$(VISUALC_FILES): $(gen_file_dep) scripts/data_files/vs2013-app-template.vcxproj +$(VISUALC_FILES): $(gen_file_dep) scripts/data_files/vs2013-main-template.vcxproj +$(VISUALC_FILES): $(gen_file_dep) scripts/data_files/vs2013-sln-template.sln # TODO: also the list of .c and .h source files, but not their content $(VISUALC_FILES): echo " Gen $@ ..." diff --git a/library/Makefile b/library/Makefile index c383c3231..69ccbfd2c 100644 --- a/library/Makefile +++ b/library/Makefile @@ -167,7 +167,7 @@ OBJS_X509= \ x509_crl.o \ x509_crt.o \ x509_csr.o \ - x509write.o \ + x509write.o \ x509write_crt.o \ x509write_csr.o \ pkcs7.o \ @@ -315,21 +315,29 @@ GENERATED_FILES = \ psa_crypto_driver_wrappers.c generated_files: $(GENERATED_FILES) -error.c: ../scripts/generate_errors.pl -error.c: ../scripts/data_files/error.fmt -error.c: $(filter-out %config%,$(wildcard ../include/mbedtls/*.h)) +# See root Makefile +GEN_FILES ?= yes +ifdef GEN_FILES +gen_file_dep = +else +gen_file_dep = | +endif + +error.c: $(gen_file_dep) ../scripts/generate_errors.pl +error.c: $(gen_file_dep) ../scripts/data_files/error.fmt +error.c: $(gen_file_dep) $(filter-out %config%,$(wildcard ../include/mbedtls/*.h)) error.c: echo " Gen $@" $(PERL) ../scripts/generate_errors.pl -ssl_debug_helpers_generated.c: ../scripts/generate_ssl_debug_helpers.py -ssl_debug_helpers_generated.c: $(filter-out %config%,$(wildcard ../include/mbedtls/*.h)) +ssl_debug_helpers_generated.c: $(gen_file_dep) ../scripts/generate_ssl_debug_helpers.py +ssl_debug_helpers_generated.c: $(gen_file_dep) $(filter-out %config%,$(wildcard ../include/mbedtls/*.h)) ssl_debug_helpers_generated.c: echo " Gen $@" $(PYTHON) ../scripts/generate_ssl_debug_helpers.py --mbedtls-root .. . -version_features.c: ../scripts/generate_features.pl -version_features.c: ../scripts/data_files/version_features.fmt +version_features.c: $(gen_file_dep) ../scripts/generate_features.pl +version_features.c: $(gen_file_dep) ../scripts/data_files/version_features.fmt ## The generated file only depends on the options that are present in mbedtls_config.h, ## not on which options are set. To avoid regenerating this file all the time ## when switching between configurations, don't declare mbedtls_config.h as a @@ -340,8 +348,8 @@ version_features.c: echo " Gen $@" $(PERL) ../scripts/generate_features.pl -psa_crypto_driver_wrappers.c: ../scripts/generate_driver_wrappers.py -psa_crypto_driver_wrappers.c: ../scripts/data_files/driver_templates/psa_crypto_driver_wrappers.c.jinja +psa_crypto_driver_wrappers.c: $(gen_file_dep) ../scripts/generate_driver_wrappers.py +psa_crypto_driver_wrappers.c: $(gen_file_dep) ../scripts/data_files/driver_templates/psa_crypto_driver_wrappers.c.jinja psa_crypto_driver_wrappers.c: echo " Gen $@" $(PYTHON) ../scripts/generate_driver_wrappers.py diff --git a/programs/Makefile b/programs/Makefile index 3509fc374..1dfebb538 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -176,22 +176,32 @@ ${MBEDTLS_TEST_OBJS}: GENERATED_FILES = psa/psa_constant_names_generated.c test/query_config.c generated_files: $(GENERATED_FILES) -psa/psa_constant_names_generated.c: ../scripts/generate_psa_constants.py -psa/psa_constant_names_generated.c: ../include/psa/crypto_values.h -psa/psa_constant_names_generated.c: ../include/psa/crypto_extra.h -psa/psa_constant_names_generated.c: ../tests/suites/test_suite_psa_crypto_metadata.data +# See root Makefile +GEN_FILES ?= yes +ifdef GEN_FILES +gen_file_dep = +else +# Order-only dependency: generate the target if it's absent, but don't +# re-generate it if it's present but older than its dependencies. +gen_file_dep = | +endif + +psa/psa_constant_names_generated.c: $(gen_file_dep) ../scripts/generate_psa_constants.py +psa/psa_constant_names_generated.c: $(gen_file_dep) ../include/psa/crypto_values.h +psa/psa_constant_names_generated.c: $(gen_file_dep) ../include/psa/crypto_extra.h +psa/psa_constant_names_generated.c: $(gen_file_dep) ../tests/suites/test_suite_psa_crypto_metadata.data psa/psa_constant_names_generated.c: echo " Gen $@" $(PYTHON) ../scripts/generate_psa_constants.py -test/query_config.c: ../scripts/generate_query_config.pl +test/query_config.c: $(gen_file_dep) ../scripts/generate_query_config.pl ## The generated file only depends on the options that are present in mbedtls_config.h, ## not on which options are set. To avoid regenerating this file all the time ## when switching between configurations, don't declare mbedtls_config.h as a ## dependency. Remove this file from your working tree if you've just added or ## removed an option in mbedtls_config.h. -#test/query_config.c: ../include/mbedtls/mbedtls_config.h -test/query_config.c: ../scripts/data_files/query_config.fmt +#test/query_config.c: $(gen_file_dep) ../include/mbedtls/mbedtls_config.h +test/query_config.c: $(gen_file_dep) ../scripts/data_files/query_config.fmt test/query_config.c: echo " Gen $@" $(PERL) ../scripts/generate_query_config.pl diff --git a/tests/Makefile b/tests/Makefile index ec016d871..60ab27ec7 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -65,6 +65,14 @@ else PYTHON ?= $(shell if type python3 >/dev/null 2>/dev/null; then echo python3; else echo python; fi) endif +# See root Makefile +GEN_FILES ?= yes +ifdef GEN_FILES +gen_file_dep = +else +gen_file_dep = | +endif + .PHONY: generated_files GENERATED_BIGNUM_DATA_FILES := $(patsubst tests/%,%,$(shell \ $(PYTHON) scripts/generate_bignum_tests.py --list || \ @@ -97,7 +105,7 @@ generated_files: $(GENERATED_FILES) # Use an intermediate phony dependency so that parallel builds don't run # a separate instance of the recipe for each output file. .SECONDARY: generated_bignum_test_data generated_ecp_test_data generated_psa_test_data -$(GENERATED_BIGNUM_DATA_FILES): generated_bignum_test_data +$(GENERATED_BIGNUM_DATA_FILES): $(gen_file_dep) generated_bignum_test_data generated_bignum_test_data: scripts/generate_bignum_tests.py generated_bignum_test_data: ../scripts/mbedtls_dev/bignum_common.py generated_bignum_test_data: ../scripts/mbedtls_dev/bignum_core.py @@ -109,7 +117,7 @@ generated_bignum_test_data: echo " Gen $(GENERATED_BIGNUM_DATA_FILES)" $(PYTHON) scripts/generate_bignum_tests.py -$(GENERATED_ECP_DATA_FILES): generated_ecp_test_data +$(GENERATED_ECP_DATA_FILES): $(gen_file_dep) generated_ecp_test_data generated_ecp_test_data: scripts/generate_ecp_tests.py generated_ecp_test_data: ../scripts/mbedtls_dev/bignum_common.py generated_ecp_test_data: ../scripts/mbedtls_dev/ecp.py @@ -119,7 +127,7 @@ generated_ecp_test_data: echo " Gen $(GENERATED_ECP_DATA_FILES)" $(PYTHON) scripts/generate_ecp_tests.py -$(GENERATED_PSA_DATA_FILES): generated_psa_test_data +$(GENERATED_PSA_DATA_FILES): $(gen_file_dep) generated_psa_test_data generated_psa_test_data: scripts/generate_psa_tests.py generated_psa_test_data: ../scripts/mbedtls_dev/crypto_data_tests.py generated_psa_test_data: ../scripts/mbedtls_dev/crypto_knowledge.py From dea4c7e09ff117590e13b96e780ab64770a7b70b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 8 Sep 2023 16:34:01 +0200 Subject: [PATCH 2/6] Prepare to generalize the script to do more than .gitignore Signed-off-by: Gilles Peskine --- scripts/{gitignore_patch.sh => prepare_release.sh} | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) rename scripts/{gitignore_patch.sh => prepare_release.sh} (92%) diff --git a/scripts/gitignore_patch.sh b/scripts/prepare_release.sh similarity index 92% rename from scripts/gitignore_patch.sh rename to scripts/prepare_release.sh index 74ec66c1d..6e6a1b779 100755 --- a/scripts/gitignore_patch.sh +++ b/scripts/prepare_release.sh @@ -1,5 +1,7 @@ #!/bin/bash -# + +# Prepare .gitignore for a release. + # Copyright The Mbed TLS Contributors # SPDX-License-Identifier: Apache-2.0 # @@ -14,13 +16,6 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Purpose -# -# For adapting gitignore files for releases so generated files can be included. -# -# Usage: gitignore_add_generated_files.sh [ -h | --help ] etc -# set -eu From 473f6363f1b5600dd3ad616a35ee2a1df64d5f93 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 8 Sep 2023 16:49:14 +0200 Subject: [PATCH 3/6] In releases, turn off GEN_FILES in Makefile and CMakeLists.txt Signed-off-by: Gilles Peskine --- scripts/prepare_release.sh | 78 +++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 31 deletions(-) diff --git a/scripts/prepare_release.sh b/scripts/prepare_release.sh index 6e6a1b779..9013cc878 100755 --- a/scripts/prepare_release.sh +++ b/scripts/prepare_release.sh @@ -1,6 +1,15 @@ #!/bin/bash -# Prepare .gitignore for a release. +print_usage() +{ + cat <&2 "$0: unknown option: -$OPTLET" + echo 1>&2 "Try '$0 --help' for more information." + exit 3;; + esac +done + + + +#### .gitignore processing #### GITIGNORES=$(find . -name ".gitignore") for GITIGNORE in $GITIGNORES; do - if $IGNORE; then + if [ -n "$unrelease" ]; then sed -i '/###START_COMMENTED_GENERATED_FILES###/,/###END_COMMENTED_GENERATED_FILES###/s/^# //' $GITIGNORE sed -i 's/###START_COMMENTED_GENERATED_FILES###/###START_GENERATED_FILES###/' $GITIGNORE sed -i 's/###END_COMMENTED_GENERATED_FILES###/###END_GENERATED_FILES###/' $GITIGNORE @@ -64,3 +60,23 @@ for GITIGNORE in $GITIGNORES; do sed -i 's/###END_GENERATED_FILES###/###END_COMMENTED_GENERATED_FILES###/' $GITIGNORE fi done + + + +#### Build scripts #### + +# GEN_FILES defaults on (non-empty) in development, off (empty) in releases +if [ -n "$unrelease" ]; then + r=' yes' +else + r='' +fi +sed -i 's/^\(GEN_FILES[ ?:]*=\)\([^#]*\)/\1'"$r/" Makefile */Makefile + +# GEN_FILES defaults on in development, off in releases +if [ -n "$unrelease" ]; then + r='ON' +else + r='OFF' +fi +sed -i 's/^\( *option *( *GEN_FILES *"[^"]*" *\)\([A-Za-z0-9][A-Za-z0-9]*\)/\1'"$r/" CMakeLists.txt From eeed74e697c7ad2b568edef36dd2e0b86f4b36f5 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sat, 9 Sep 2023 21:00:30 +0200 Subject: [PATCH 4/6] Fix `prepare_release.sh -u` with CMakeLists.txt We want to keep GEN_FILES defaulting off on Windows. Signed-off-by: Gilles Peskine --- CMakeLists.txt | 3 ++- scripts/prepare_release.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4d7e0b055..a99ca38b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,7 +57,8 @@ option(ENABLE_PROGRAMS "Build mbed TLS programs." ON) option(UNSAFE_BUILD "Allow unsafe builds. These builds ARE NOT SECURE." OFF) option(MBEDTLS_FATAL_WARNINGS "Compiler warnings treated as errors" ON) if(CMAKE_HOST_WIN32) - option(GEN_FILES "Generate the auto-generated files as needed" OFF) + # N.B. The comment on the next line is significant for prepare_release.sh + option(GEN_FILES "Generate the auto-generated files as needed" OFF) # off in development else() option(GEN_FILES "Generate the auto-generated files as needed" ON) endif() diff --git a/scripts/prepare_release.sh b/scripts/prepare_release.sh index 9013cc878..800383d2c 100755 --- a/scripts/prepare_release.sh +++ b/scripts/prepare_release.sh @@ -79,4 +79,4 @@ if [ -n "$unrelease" ]; then else r='OFF' fi -sed -i 's/^\( *option *( *GEN_FILES *"[^"]*" *\)\([A-Za-z0-9][A-Za-z0-9]*\)/\1'"$r/" CMakeLists.txt +sed -i '/[Oo][Ff][Ff] in development/! s/^\( *option *( *GEN_FILES *"[^"]*" *\)\([A-Za-z0-9][A-Za-z0-9]*\)/\1'"$r/" CMakeLists.txt From 1bfc09bca74f1f6468620c5bc01cccb85f3b90e7 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sat, 9 Sep 2023 21:26:05 +0200 Subject: [PATCH 5/6] Move comment to its intended location Signed-off-by: Gilles Peskine --- Makefile | 2 ++ programs/Makefile | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b324b9a35..885948c11 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,8 @@ GEN_FILES ?= yes ifdef GEN_FILES gen_file_dep = else +# Order-only dependency: generate the target if it's absent, but don't +# re-generate it if it's present but older than its dependencies. gen_file_dep = | endif diff --git a/programs/Makefile b/programs/Makefile index 1dfebb538..5f47e25bc 100644 --- a/programs/Makefile +++ b/programs/Makefile @@ -181,8 +181,6 @@ GEN_FILES ?= yes ifdef GEN_FILES gen_file_dep = else -# Order-only dependency: generate the target if it's absent, but don't -# re-generate it if it's present but older than its dependencies. gen_file_dep = | endif From ecc99611e4db352eed8d20f7e027afe81920a1a6 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sat, 9 Sep 2023 22:54:26 +0200 Subject: [PATCH 6/6] Reinforce warning about a significant comment Signed-off-by: Gilles Peskine --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a99ca38b1..c5c9a24f4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,7 +57,9 @@ option(ENABLE_PROGRAMS "Build mbed TLS programs." ON) option(UNSAFE_BUILD "Allow unsafe builds. These builds ARE NOT SECURE." OFF) option(MBEDTLS_FATAL_WARNINGS "Compiler warnings treated as errors" ON) if(CMAKE_HOST_WIN32) - # N.B. The comment on the next line is significant for prepare_release.sh + # N.B. The comment on the next line is significant! If you change it, + # edit the sed command in prepare_release.sh that modifies + # CMakeLists.txt. option(GEN_FILES "Generate the auto-generated files as needed" OFF) # off in development else() option(GEN_FILES "Generate the auto-generated files as needed" ON)