From 45137617db2f7f176aea5f7b59b87447b754d8aa Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 10 May 2020 16:52:44 +0200 Subject: [PATCH 01/12] More accurate variable name Signed-off-by: Gilles Peskine --- tests/scripts/check-files.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/scripts/check-files.py b/tests/scripts/check-files.py index 2cf608100..069afd059 100755 --- a/tests/scripts/check-files.py +++ b/tests/scripts/check-files.py @@ -23,13 +23,13 @@ class FileIssueTracker: To implement a checker that processes a file as a whole, inherit from this class and implement `check_file_for_issue` and define ``heading``. - ``files_exemptions``: files whose name ends with a string in this set + ``suffix_exemptions``: files whose name ends with a string in this set will not be checked. ``heading``: human-readable description of the issue """ - files_exemptions = frozenset() + suffix_exemptions = frozenset() # heading must be defined in derived classes. # pylint: disable=no-member @@ -39,10 +39,10 @@ class FileIssueTracker: def should_check_file(self, filepath): """Whether the given file name should be checked. - Files whose name ends with a string listed in ``self.files_exemptions`` - will not be checked. + Files whose name ends with a string listed in ``self.suffix_exemptions`` + or whose path matches ``self.path_exemptions`` will not be checked. """ - for files_exemption in self.files_exemptions: + for files_exemption in self.suffix_exemptions: if filepath.endswith(files_exemption): return False return True @@ -138,7 +138,7 @@ class Utf8BomIssueTracker(FileIssueTracker): heading = "UTF-8 BOM present:" - files_exemptions = frozenset([".vcxproj", ".sln"]) + suffix_exemptions = frozenset([".vcxproj", ".sln"]) def check_file_for_issue(self, filepath): with open(filepath, "rb") as f: @@ -174,7 +174,7 @@ class TrailingWhitespaceIssueTracker(LineIssueTracker): """Track lines with trailing whitespace.""" heading = "Trailing whitespace:" - files_exemptions = frozenset([".dsp", ".md"]) + suffix_exemptions = frozenset([".dsp", ".md"]) def issue_with_line(self, line, _filepath): return line.rstrip(b"\r\n") != line.rstrip() @@ -184,7 +184,7 @@ class TabIssueTracker(LineIssueTracker): """Track lines with tabs.""" heading = "Tabs present:" - files_exemptions = frozenset([ + suffix_exemptions = frozenset([ ".sln", "/Makefile", "/generate_visualc_files.pl", From ab55bc71fec9c8efd83b9d446d81a398cef6fc54 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 10 May 2020 16:54:10 +0200 Subject: [PATCH 02/12] Check all files by default Have an explicit list of exemptions for specific checks rather than whitelisting files to check. Some checks, such as permissions, should apply to all files. Signed-off-by: Gilles Peskine --- tests/scripts/check-files.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/tests/scripts/check-files.py b/tests/scripts/check-files.py index 069afd059..11073f5e9 100755 --- a/tests/scripts/check-files.py +++ b/tests/scripts/check-files.py @@ -222,23 +222,6 @@ class IntegrityChecker: self.check_repo_path() self.logger = None self.setup_logger(log_file) - self.extensions_to_check = ( - ".bat", - ".c", - ".data", - ".dsp", - ".function", - ".h", - ".md", - ".pl", - ".py", - ".sh", - ".sln", - ".vcxproj", - "/CMakeLists.txt", - "/ChangeLog", - "/Makefile", - ) self.excluded_directories = [ '.git', 'mbed-os', @@ -286,8 +269,6 @@ class IntegrityChecker: dirs[:] = sorted(d for d in dirs if not self.prune_branch(root, d)) for filename in sorted(files): filepath = os.path.join(root, filename) - if not filepath.endswith(self.extensions_to_check): - continue for issue_to_check in self.issues_to_check: if issue_to_check.should_check_file(filepath): issue_to_check.check_file_for_issue(filepath) From b4805ece7844c6b4b039ee41c4c81d3a67e2ee1e Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 10 May 2020 16:57:16 +0200 Subject: [PATCH 03/12] Regex mechanism for check-specific exemptions Suffixes are convenient but not always sufficient. Signed-off-by: Gilles Peskine --- tests/scripts/check-files.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/scripts/check-files.py b/tests/scripts/check-files.py index 11073f5e9..3992c4668 100755 --- a/tests/scripts/check-files.py +++ b/tests/scripts/check-files.py @@ -14,6 +14,7 @@ import os import argparse import logging import codecs +import re import sys @@ -26,16 +27,31 @@ class FileIssueTracker: ``suffix_exemptions``: files whose name ends with a string in this set will not be checked. + ``path_exemptions``: files whose path (relative to the root of the source + tree) matches this regular expression will not be checked. This can be + ``None`` to match no path. Paths are normalized and converted to ``/`` + separators before matching. + ``heading``: human-readable description of the issue """ suffix_exemptions = frozenset() + path_exemptions = None # heading must be defined in derived classes. # pylint: disable=no-member def __init__(self): self.files_with_issues = {} + @staticmethod + def normalize_path(filepath): + """Normalize ``filepath`` """ + filepath = os.path.normpath(filepath) + seps = os.path.sep + if os.path.altsep is not None: + seps += os.path.altsep + return '/'.join(filepath.split(seps)) + def should_check_file(self, filepath): """Whether the given file name should be checked. @@ -45,6 +61,9 @@ class FileIssueTracker: for files_exemption in self.suffix_exemptions: if filepath.endswith(files_exemption): return False + if self.path_exemptions and \ + re.match(self.path_exemptions, self.normalize_path(filepath)): + return False return True def check_file_for_issue(self, filepath): @@ -152,6 +171,8 @@ class UnixLineEndingIssueTracker(LineIssueTracker): heading = "Non-Unix line endings:" def should_check_file(self, filepath): + if not super().should_check_file(filepath): + return False return not is_windows_file(filepath) def issue_with_line(self, line, _filepath): @@ -164,6 +185,8 @@ class WindowsLineEndingIssueTracker(LineIssueTracker): heading = "Non-Windows line endings:" def should_check_file(self, filepath): + if not super().should_check_file(filepath): + return False return is_windows_file(filepath) def issue_with_line(self, line, _filepath): From 986a06de94f9e5e6eed46dbaf73a68b3036cf500 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 10 May 2020 16:57:59 +0200 Subject: [PATCH 04/12] Exclude binary files from text checks Signed-off-by: Gilles Peskine --- tests/scripts/check-files.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/scripts/check-files.py b/tests/scripts/check-files.py index 3992c4668..b48f62b75 100755 --- a/tests/scripts/check-files.py +++ b/tests/scripts/check-files.py @@ -92,6 +92,17 @@ class FileIssueTracker: logger.info(filename) logger.info("") +BINARY_FILE_PATH_RE_LIST = [ + r'docs/.*\.pdf\Z', + r'programs/fuzz/corpuses/[^.]+\Z', + r'tests/data_files/[^.]+\Z', + r'tests/data_files/.*\.(crt|csr|db|der|key|pubkey)\Z', + r'tests/data_files/.*\.req\.[^/]+\Z', + r'tests/data_files/.*malformed[^/]+\Z', + r'tests/data_files/format_pkcs12\.fmt\Z', +] +BINARY_FILE_PATH_RE = re.compile('|'.join(BINARY_FILE_PATH_RE_LIST)) + class LineIssueTracker(FileIssueTracker): """Base class for line-by-line issue tracking. @@ -99,6 +110,9 @@ class LineIssueTracker(FileIssueTracker): this class and implement `line_with_issue`. """ + # Exclude binary files. + path_exemptions = BINARY_FILE_PATH_RE + def issue_with_line(self, line, filepath): """Check the specified line for the issue that this class is for. @@ -145,6 +159,8 @@ class EndOfFileNewlineIssueTracker(FileIssueTracker): heading = "Missing newline at end of file:" + path_exemptions = BINARY_FILE_PATH_RE + def check_file_for_issue(self, filepath): with open(filepath, "rb") as f: if not f.read().endswith(b"\n"): @@ -158,6 +174,7 @@ class Utf8BomIssueTracker(FileIssueTracker): heading = "UTF-8 BOM present:" suffix_exemptions = frozenset([".vcxproj", ".sln"]) + path_exemptions = BINARY_FILE_PATH_RE def check_file_for_issue(self, filepath): with open(filepath, "rb") as f: From 4bda3254cf29651abc9deaff4816cbdb45e8c1c0 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 10 May 2020 17:18:06 +0200 Subject: [PATCH 05/12] Check only files checked into Git We're only interested in files that are committed and pushed to be included in Mbed TLS, not in any other files that may be lying around. So ask git for the list of file names. This script is primarily intended to run on the CI, and there it runs on a fresh Git checkout plus potentially some other checkouts or leftovers from a previous part of the CI job. It should also run reasonably well on developer machines, where there may be various additional files. In both cases, git is available. Ad hoc directory exclusions are no longer needed. Signed-off-by: Gilles Peskine --- tests/scripts/check-files.py | 37 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/tests/scripts/check-files.py b/tests/scripts/check-files.py index b48f62b75..39a76931b 100755 --- a/tests/scripts/check-files.py +++ b/tests/scripts/check-files.py @@ -15,6 +15,7 @@ import argparse import logging import codecs import re +import subprocess import sys @@ -262,15 +263,6 @@ class IntegrityChecker: self.check_repo_path() self.logger = None self.setup_logger(log_file) - self.excluded_directories = [ - '.git', - 'mbed-os', - ] - self.excluded_paths = list(map(os.path.normpath, [ - 'cov-int', - 'examples', - 'yotta/module' - ])) self.issues_to_check = [ PermissionIssueTracker(), EndOfFileNewlineIssueTracker(), @@ -297,21 +289,22 @@ class IntegrityChecker: console = logging.StreamHandler() self.logger.addHandler(console) - def prune_branch(self, root, d): - if d in self.excluded_directories: - return True - if os.path.normpath(os.path.join(root, d)) in self.excluded_paths: - return True - return False + @staticmethod + def collect_files(): + bytes_output = subprocess.check_output(['git', 'ls-files', '-z']) + bytes_filepaths = bytes_output.split(b'\0')[:-1] + ascii_filepaths = map(lambda fp: fp.decode('ascii'), bytes_filepaths) + # Prepend './' to files in the top-level directory so that + # something like `'/Makefile' in fp` matches in the top-level + # directory as well as in subdirectories. + return [fp if os.path.dirname(fp) else os.path.join(os.curdir, fp) + for fp in ascii_filepaths] def check_files(self): - for root, dirs, files in os.walk("."): - dirs[:] = sorted(d for d in dirs if not self.prune_branch(root, d)) - for filename in sorted(files): - filepath = os.path.join(root, filename) - for issue_to_check in self.issues_to_check: - if issue_to_check.should_check_file(filepath): - issue_to_check.check_file_for_issue(filepath) + for issue_to_check in self.issues_to_check: + for filepath in self.collect_files(): + if issue_to_check.should_check_file(filepath): + issue_to_check.check_file_for_issue(filepath) def output_issues(self): integrity_return_code = 0 From 4be18cf1f1134788f420cae98f50d5139ab4c42b Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 10 May 2020 17:25:39 +0200 Subject: [PATCH 06/12] Normalize line endings Convert all text files to Unix line endings unless they're Windows stuff. Make sure that all text files have a trailing newline. Remove whitespace at the end of lines. Signed-off-by: Gilles Peskine --- ChangeLog.d/bugfix.txt | 2 +- programs/pkey/rsa_priv.txt | 16 +-- programs/pkey/rsa_pub.txt | 4 +- tests/data_files/bitstring-in-dn.pem | 102 +++++++++---------- tests/data_files/test-ca.server1.opensslconf | 2 +- 5 files changed, 63 insertions(+), 63 deletions(-) diff --git a/ChangeLog.d/bugfix.txt b/ChangeLog.d/bugfix.txt index 6e2cea1f6..b7d4cd688 100644 --- a/ChangeLog.d/bugfix.txt +++ b/ChangeLog.d/bugfix.txt @@ -1,4 +1,4 @@ Bugfix * Fix the Visual Studio Release x64 build configuration for mbedtls itself. Completes a previous fix in Mbed TLS 2.7 that only fixed the build for - the example programs. Reported in #1430 and fix contributed by irwir. \ No newline at end of file + the example programs. Reported in #1430 and fix contributed by irwir. diff --git a/programs/pkey/rsa_priv.txt b/programs/pkey/rsa_priv.txt index 22c37fe61..254fcf852 100644 --- a/programs/pkey/rsa_priv.txt +++ b/programs/pkey/rsa_priv.txt @@ -1,8 +1,8 @@ -N = A1D46FBA2318F8DCEF16C280948B1CF27966B9B47225ED2989F8D74B45BD36049C0AAB5AD0FF003553BA843C8E12782FC5873BB89A3DC84B883D25666CD22BF3ACD5B675969F8BEBFBCAC93FDD927C7442B178B10D1DFF9398E52316AAE0AF74E594650BDC3C670241D418684593CDA1A7B9DC4F20D2FDC6F66344074003E211 -E = 010001 -D = 589552BB4F2F023ADDDD5586D0C8FD857512D82080436678D07F984A29D892D31F1F7000FC5A39A0F73E27D885E47249A4148C8A5653EF69F91F8F736BA9F84841C2D99CD8C24DE8B72B5C9BE0EDBE23F93D731749FEA9CFB4A48DD2B7F35A2703E74AA2D4DB7DE9CEEA7D763AF0ADA7AC176C4E9A22C4CDA65CEC0C65964401 -P = CD083568D2D46C44C40C1FA0101AF2155E59C70B08423112AF0C1202514BBA5210765E29FF13036F56C7495894D80CF8C3BAEE2839BACBB0B86F6A2965F60DB1 -Q = CA0EEEA5E710E8E9811A6B846399420E3AE4A4C16647E426DDF8BBBCB11CD3F35CE2E4B6BCAD07AE2C0EC2ECBFCC601B207CDD77B5673E16382B1130BF465261 -DP = 0D0E21C07BF434B4A83B116472C2147A11D8EB98A33CFBBCF1D275EF19D815941622435AAF3839B6C432CA53CE9E772CFBE1923A937A766FD93E96E6EDEC1DF1 -DQ = 269CEBE6305DFEE4809377F078C814E37B45AE6677114DFC4F76F5097E1F3031D592567AC55B9B98213B40ECD54A4D2361F5FAACA1B1F51F71E4690893C4F081 -QP = 97AC5BB885ABCA314375E9E4DB1BA4B2218C90619F61BD474F5785075ECA81750A735199A8C191FE2D3355E7CF601A70E5CABDE0E02C2538BB9FB4871540B3C1 +N = A1D46FBA2318F8DCEF16C280948B1CF27966B9B47225ED2989F8D74B45BD36049C0AAB5AD0FF003553BA843C8E12782FC5873BB89A3DC84B883D25666CD22BF3ACD5B675969F8BEBFBCAC93FDD927C7442B178B10D1DFF9398E52316AAE0AF74E594650BDC3C670241D418684593CDA1A7B9DC4F20D2FDC6F66344074003E211 +E = 010001 +D = 589552BB4F2F023ADDDD5586D0C8FD857512D82080436678D07F984A29D892D31F1F7000FC5A39A0F73E27D885E47249A4148C8A5653EF69F91F8F736BA9F84841C2D99CD8C24DE8B72B5C9BE0EDBE23F93D731749FEA9CFB4A48DD2B7F35A2703E74AA2D4DB7DE9CEEA7D763AF0ADA7AC176C4E9A22C4CDA65CEC0C65964401 +P = CD083568D2D46C44C40C1FA0101AF2155E59C70B08423112AF0C1202514BBA5210765E29FF13036F56C7495894D80CF8C3BAEE2839BACBB0B86F6A2965F60DB1 +Q = CA0EEEA5E710E8E9811A6B846399420E3AE4A4C16647E426DDF8BBBCB11CD3F35CE2E4B6BCAD07AE2C0EC2ECBFCC601B207CDD77B5673E16382B1130BF465261 +DP = 0D0E21C07BF434B4A83B116472C2147A11D8EB98A33CFBBCF1D275EF19D815941622435AAF3839B6C432CA53CE9E772CFBE1923A937A766FD93E96E6EDEC1DF1 +DQ = 269CEBE6305DFEE4809377F078C814E37B45AE6677114DFC4F76F5097E1F3031D592567AC55B9B98213B40ECD54A4D2361F5FAACA1B1F51F71E4690893C4F081 +QP = 97AC5BB885ABCA314375E9E4DB1BA4B2218C90619F61BD474F5785075ECA81750A735199A8C191FE2D3355E7CF601A70E5CABDE0E02C2538BB9FB4871540B3C1 diff --git a/programs/pkey/rsa_pub.txt b/programs/pkey/rsa_pub.txt index 2c6d313af..1e7ae0c9c 100644 --- a/programs/pkey/rsa_pub.txt +++ b/programs/pkey/rsa_pub.txt @@ -1,2 +1,2 @@ -N = A1D46FBA2318F8DCEF16C280948B1CF27966B9B47225ED2989F8D74B45BD36049C0AAB5AD0FF003553BA843C8E12782FC5873BB89A3DC84B883D25666CD22BF3ACD5B675969F8BEBFBCAC93FDD927C7442B178B10D1DFF9398E52316AAE0AF74E594650BDC3C670241D418684593CDA1A7B9DC4F20D2FDC6F66344074003E211 -E = 010001 +N = A1D46FBA2318F8DCEF16C280948B1CF27966B9B47225ED2989F8D74B45BD36049C0AAB5AD0FF003553BA843C8E12782FC5873BB89A3DC84B883D25666CD22BF3ACD5B675969F8BEBFBCAC93FDD927C7442B178B10D1DFF9398E52316AAE0AF74E594650BDC3C670241D418684593CDA1A7B9DC4F20D2FDC6F66344074003E211 +E = 010001 diff --git a/tests/data_files/bitstring-in-dn.pem b/tests/data_files/bitstring-in-dn.pem index 1a98aa3ac..c50bd6684 100644 --- a/tests/data_files/bitstring-in-dn.pem +++ b/tests/data_files/bitstring-in-dn.pem @@ -1,51 +1,51 @@ ------BEGIN CERTIFICATE----- -MIIEATCCAumgAwIBAgIBAjANBgkqhkiG9w0BAQUFADBxMRMwEQYDVQQDDApUZXN0 -IENBIDAxMREwDwYDVQQIDAhFY25pdm9ycDELMAkGA1UEBhMCWFgxHjAcBgkqhkiG -9w0BCQEWD3RjYUBleGFtcGxlLmNvbTEaMBgGA1UECgwRVGVzdCBDQSBBdXRob3Jp -dHkwHhcNMTUwMzExMTIwNjUxWhcNMjUwMzA4MTIwNjUxWjCBmzELMAkGA1UEBhMC -WFgxDDAKBgNVBAoMA3RjYTERMA8GA1UECAwIRWNuaXZvcnAxDDAKBgNVBAsMA1RD -QTEPMA0GA1UEAwwGQ2xpZW50MSEwHwYJKoZIhvcNAQkBFhJjbGllbnRAZXhhbXBs -ZS5jb20xEzARBgNVBAUTCjcxMDEwMTIyNTUxFDASBgNVBC0DCwA3MTAxMDEyMjU1 -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnQS0JLb8Dqy8V2mszkWk -V8c/NPQcG3ivueXZHqOT9JTiPqrigGcLHtlmlaJ0aUUxix7q60aOds041TFyeknT -SUFYY4ppOhiP+fOpWKPv4ZMwhSI2XzcgYhQSNHV0lIG1we9RAAfumomDMq7oMJhb -EGf0ihibbwZXPUwBlm10GaB4K93PNY8Bz4ekBxzQ1WJkQ5LGsQnVZSuLnvp5dWSe -J2axxyY4hPXR30jzEyZvy4kv4nzAu5lqZ5XKLrRO4TKwZrtr+CCPVkPJRE36rWYt -tQaJEEpNOo0ZPpTtG6F8/tGh5r8jFx/f6wG+nyANJJ98kEP8i6TPjRrg+697mLcd -iQIDAQABo3kwdzAJBgNVHRMEAjAAMDYGA1UdHwQvMC0wK6ApoCeGJWh0dHA6Ly9j -cmwuZXhhbXBsZS5jb20vdGVzdF9jYV8wMS5jcmwwEwYDVR0lBAwwCgYIKwYBBQUH -AwIwHQYDVR0RBBYwFIESY2xpZW50QGV4YW1wbGUuY29tMA0GCSqGSIb3DQEBBQUA -A4IBAQBySELCnU8/PtGIG3dwhJENOSU5R7w8jpRXxHCuSBR+W6nuUCISz+z+EdF/ -A7AOJDASuS+4gkrSSmQhGFpf7E5VbF8trVZhLAZrXqKMcUreKH6v0I8MAUXmIs3G -tqiBGf7pSYJN9DvVOOgANjdy6THuUzYv5qSvBZ4pNYEfHSlMNrV7niynd8dgPOML -pA7GUfv5k2mMkMbSD15pTMgcavrBKYgyqcvF1C3qghfoL5+i38H8sKzF8hy7wHtE -ESHtBq20RYA3m0UcA0e64GcanO2Ps/AQVBc7qMeHbqnqj3uUhtTkQcMUWnMgy1NR -5RbzoLMOxq7hoOCyIaQeM/wgxeGE ------END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAnQS0JLb8Dqy8V2mszkWkV8c/NPQcG3ivueXZHqOT9JTiPqri -gGcLHtlmlaJ0aUUxix7q60aOds041TFyeknTSUFYY4ppOhiP+fOpWKPv4ZMwhSI2 -XzcgYhQSNHV0lIG1we9RAAfumomDMq7oMJhbEGf0ihibbwZXPUwBlm10GaB4K93P -NY8Bz4ekBxzQ1WJkQ5LGsQnVZSuLnvp5dWSeJ2axxyY4hPXR30jzEyZvy4kv4nzA -u5lqZ5XKLrRO4TKwZrtr+CCPVkPJRE36rWYttQaJEEpNOo0ZPpTtG6F8/tGh5r8j -Fx/f6wG+nyANJJ98kEP8i6TPjRrg+697mLcdiQIDAQABAoIBAF7i3MnjGmbz080v -OxJb23iAG54wdlvTjr3UPGTbjSmcXyxnsADQRFQcJHYAekCzY8EiqewL80OvuMx8 -2SU1P81hA70Dg5tsBHWT3Z6HUwsKG6QYjKr1cUhTwLyazhyAVgogSN6v7GzO9M3I -DOBw8Xb0mz5oqGVre4S7TapN8n8ZG5oWm0XKGACXy0KbzY0KvWdkUzumFQ8X/ARE -FsWyu+O69EbMqZRUKu45SrcubsdVGjOwseZHkmp5V6pc6Q/OrTHZqXJtDva5UIRq -+Lof5scy9jiwwRnM/klvh23mz0ySU4YA3645m5KqyWR4YJCR1MnMANmXUSeYWfYz -19+R1gECgYEAzm83lI7eIhTH38H0/jFpf3R7vNjPX3TR5waa4EXsCxhTOpoL89mR -iNmzH0aOC4OR8rz/9PCnwmtH1lyQ4r/RokBmCp3pBxeWSlenFfV3rLCeEDo0Q/OL -SX5DL4IbZD0VmNDt606WS7AEv93GhpN03Anw6kgHQUm1l030PR9DYZECgYEAwrgO -/RyB/Ehw7smlysZb2sn1lvd6z8fg+pcu8ZNRKODaYCCOb8p1lnHrnIQdEmjhlmVp -HAEuJ5jxCb+lyruV+dlx+0W/p6lHtKr0iBHG8EFkHnjN6Y+59Qu0HfSm0pZw7Ftr -QcUDPuDJkTVUAvrZqciWlwzTWCC9KYXtasT+AHkCgYEAnP80dAUbpyvKvr/RxShr -JYW/PWZegChmIp+BViOXWvDLC3xwrqm+5yc59QVBrjwH2WYn+26zB0dzwPFxNyHP -GuiDMnvZ54zmve9foXGn7Gv+KjU53pvwSJqAGjeHAXr7W9c5uoVwBGv/kLPn8h1e -+KGO2X6iFeMq+cFNiNan9iECgYBj+oGnsKWFVeS2ls8LyMGNGzmAZF2opiZ8RHgU -DeIULS+zP8Qi3j92GdQyLxuGQlfiEvvfJzP9nOfWa5LC/4JIIUAHFo8LlT1+JHEe -FJKi9dBkXP7NN8DxcyruXpnxctFUarQttuytslmMt2cFiKuOI7I+qJUzoMu/sEZx -FeidcQKBgQCuralmtbl4nxjn3aR/ZgFTAKCL9WaJPh5hFJ9q4UuWxJdBX5z3Ey3/ -70ehLKYPumjmZtXynzz4BTWj1W9X+tgj/499uzV6LdQERGjT6WVy8xR9RELWW0an -N9N1IAc4qTSjbI4EIMwMBSAoFfCux/jfDkG4g+RDnpV92sqxz2CtKg== ------END RSA PRIVATE KEY----- \ No newline at end of file +-----BEGIN CERTIFICATE----- +MIIEATCCAumgAwIBAgIBAjANBgkqhkiG9w0BAQUFADBxMRMwEQYDVQQDDApUZXN0 +IENBIDAxMREwDwYDVQQIDAhFY25pdm9ycDELMAkGA1UEBhMCWFgxHjAcBgkqhkiG +9w0BCQEWD3RjYUBleGFtcGxlLmNvbTEaMBgGA1UECgwRVGVzdCBDQSBBdXRob3Jp +dHkwHhcNMTUwMzExMTIwNjUxWhcNMjUwMzA4MTIwNjUxWjCBmzELMAkGA1UEBhMC +WFgxDDAKBgNVBAoMA3RjYTERMA8GA1UECAwIRWNuaXZvcnAxDDAKBgNVBAsMA1RD +QTEPMA0GA1UEAwwGQ2xpZW50MSEwHwYJKoZIhvcNAQkBFhJjbGllbnRAZXhhbXBs +ZS5jb20xEzARBgNVBAUTCjcxMDEwMTIyNTUxFDASBgNVBC0DCwA3MTAxMDEyMjU1 +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnQS0JLb8Dqy8V2mszkWk +V8c/NPQcG3ivueXZHqOT9JTiPqrigGcLHtlmlaJ0aUUxix7q60aOds041TFyeknT +SUFYY4ppOhiP+fOpWKPv4ZMwhSI2XzcgYhQSNHV0lIG1we9RAAfumomDMq7oMJhb +EGf0ihibbwZXPUwBlm10GaB4K93PNY8Bz4ekBxzQ1WJkQ5LGsQnVZSuLnvp5dWSe +J2axxyY4hPXR30jzEyZvy4kv4nzAu5lqZ5XKLrRO4TKwZrtr+CCPVkPJRE36rWYt +tQaJEEpNOo0ZPpTtG6F8/tGh5r8jFx/f6wG+nyANJJ98kEP8i6TPjRrg+697mLcd +iQIDAQABo3kwdzAJBgNVHRMEAjAAMDYGA1UdHwQvMC0wK6ApoCeGJWh0dHA6Ly9j +cmwuZXhhbXBsZS5jb20vdGVzdF9jYV8wMS5jcmwwEwYDVR0lBAwwCgYIKwYBBQUH +AwIwHQYDVR0RBBYwFIESY2xpZW50QGV4YW1wbGUuY29tMA0GCSqGSIb3DQEBBQUA +A4IBAQBySELCnU8/PtGIG3dwhJENOSU5R7w8jpRXxHCuSBR+W6nuUCISz+z+EdF/ +A7AOJDASuS+4gkrSSmQhGFpf7E5VbF8trVZhLAZrXqKMcUreKH6v0I8MAUXmIs3G +tqiBGf7pSYJN9DvVOOgANjdy6THuUzYv5qSvBZ4pNYEfHSlMNrV7niynd8dgPOML +pA7GUfv5k2mMkMbSD15pTMgcavrBKYgyqcvF1C3qghfoL5+i38H8sKzF8hy7wHtE +ESHtBq20RYA3m0UcA0e64GcanO2Ps/AQVBc7qMeHbqnqj3uUhtTkQcMUWnMgy1NR +5RbzoLMOxq7hoOCyIaQeM/wgxeGE +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAnQS0JLb8Dqy8V2mszkWkV8c/NPQcG3ivueXZHqOT9JTiPqri +gGcLHtlmlaJ0aUUxix7q60aOds041TFyeknTSUFYY4ppOhiP+fOpWKPv4ZMwhSI2 +XzcgYhQSNHV0lIG1we9RAAfumomDMq7oMJhbEGf0ihibbwZXPUwBlm10GaB4K93P +NY8Bz4ekBxzQ1WJkQ5LGsQnVZSuLnvp5dWSeJ2axxyY4hPXR30jzEyZvy4kv4nzA +u5lqZ5XKLrRO4TKwZrtr+CCPVkPJRE36rWYttQaJEEpNOo0ZPpTtG6F8/tGh5r8j +Fx/f6wG+nyANJJ98kEP8i6TPjRrg+697mLcdiQIDAQABAoIBAF7i3MnjGmbz080v +OxJb23iAG54wdlvTjr3UPGTbjSmcXyxnsADQRFQcJHYAekCzY8EiqewL80OvuMx8 +2SU1P81hA70Dg5tsBHWT3Z6HUwsKG6QYjKr1cUhTwLyazhyAVgogSN6v7GzO9M3I +DOBw8Xb0mz5oqGVre4S7TapN8n8ZG5oWm0XKGACXy0KbzY0KvWdkUzumFQ8X/ARE +FsWyu+O69EbMqZRUKu45SrcubsdVGjOwseZHkmp5V6pc6Q/OrTHZqXJtDva5UIRq ++Lof5scy9jiwwRnM/klvh23mz0ySU4YA3645m5KqyWR4YJCR1MnMANmXUSeYWfYz +19+R1gECgYEAzm83lI7eIhTH38H0/jFpf3R7vNjPX3TR5waa4EXsCxhTOpoL89mR +iNmzH0aOC4OR8rz/9PCnwmtH1lyQ4r/RokBmCp3pBxeWSlenFfV3rLCeEDo0Q/OL +SX5DL4IbZD0VmNDt606WS7AEv93GhpN03Anw6kgHQUm1l030PR9DYZECgYEAwrgO +/RyB/Ehw7smlysZb2sn1lvd6z8fg+pcu8ZNRKODaYCCOb8p1lnHrnIQdEmjhlmVp +HAEuJ5jxCb+lyruV+dlx+0W/p6lHtKr0iBHG8EFkHnjN6Y+59Qu0HfSm0pZw7Ftr +QcUDPuDJkTVUAvrZqciWlwzTWCC9KYXtasT+AHkCgYEAnP80dAUbpyvKvr/RxShr +JYW/PWZegChmIp+BViOXWvDLC3xwrqm+5yc59QVBrjwH2WYn+26zB0dzwPFxNyHP +GuiDMnvZ54zmve9foXGn7Gv+KjU53pvwSJqAGjeHAXr7W9c5uoVwBGv/kLPn8h1e ++KGO2X6iFeMq+cFNiNan9iECgYBj+oGnsKWFVeS2ls8LyMGNGzmAZF2opiZ8RHgU +DeIULS+zP8Qi3j92GdQyLxuGQlfiEvvfJzP9nOfWa5LC/4JIIUAHFo8LlT1+JHEe +FJKi9dBkXP7NN8DxcyruXpnxctFUarQttuytslmMt2cFiKuOI7I+qJUzoMu/sEZx +FeidcQKBgQCuralmtbl4nxjn3aR/ZgFTAKCL9WaJPh5hFJ9q4UuWxJdBX5z3Ey3/ +70ehLKYPumjmZtXynzz4BTWj1W9X+tgj/499uzV6LdQERGjT6WVy8xR9RELWW0an +N9N1IAc4qTSjbI4EIMwMBSAoFfCux/jfDkG4g+RDnpV92sqxz2CtKg== +-----END RSA PRIVATE KEY----- diff --git a/tests/data_files/test-ca.server1.opensslconf b/tests/data_files/test-ca.server1.opensslconf index 4a5072eae..209b0fffa 100644 --- a/tests/data_files/test-ca.server1.opensslconf +++ b/tests/data_files/test-ca.server1.opensslconf @@ -1,6 +1,6 @@ [ ca ] default_ca = test-ca - + [ test-ca ] certificate = test-ca.crt private_key = test-ca.key From be76c1980101e96581577ae3fdc59b73a00af1bb Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 10 May 2020 17:36:42 +0200 Subject: [PATCH 07/12] Permit empty files Signed-off-by: Gilles Peskine --- tests/scripts/check-files.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/scripts/check-files.py b/tests/scripts/check-files.py index 39a76931b..96911c8eb 100755 --- a/tests/scripts/check-files.py +++ b/tests/scripts/check-files.py @@ -164,7 +164,14 @@ class EndOfFileNewlineIssueTracker(FileIssueTracker): def check_file_for_issue(self, filepath): with open(filepath, "rb") as f: - if not f.read().endswith(b"\n"): + try: + f.seek(-1, 2) + except OSError: + # This script only works on regular files. If we can't seek + # 1 before the end, it means that this position is before + # the beginning of the file, i.e. that the file is empty. + return + if f.read(1) != b"\n": self.files_with_issues[filepath] = None From e7e149f4691cbfbf2cc539f52fa4be0da31bdbeb Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 10 May 2020 17:36:51 +0200 Subject: [PATCH 08/12] .dsw files are Visual Studio stuff Signed-off-by: Gilles Peskine --- tests/scripts/check-files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/check-files.py b/tests/scripts/check-files.py index 96911c8eb..84cad4905 100755 --- a/tests/scripts/check-files.py +++ b/tests/scripts/check-files.py @@ -137,7 +137,7 @@ class LineIssueTracker(FileIssueTracker): def is_windows_file(filepath): _root, ext = os.path.splitext(filepath) - return ext in ('.bat', '.dsp', '.sln', '.vcxproj') + return ext in ('.bat', '.dsp', '.dsw', '.sln', '.vcxproj') class PermissionIssueTracker(FileIssueTracker): From 8fa5be52241dcba5986b93db5b31bf5f07d7bd6f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 10 May 2020 17:37:02 +0200 Subject: [PATCH 09/12] Some .pem files are openssl output and have tabs and that's ok Signed-off-by: Gilles Peskine --- tests/scripts/check-files.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/scripts/check-files.py b/tests/scripts/check-files.py index 84cad4905..86be3ffb3 100755 --- a/tests/scripts/check-files.py +++ b/tests/scripts/check-files.py @@ -233,6 +233,7 @@ class TabIssueTracker(LineIssueTracker): heading = "Tabs present:" suffix_exemptions = frozenset([ + ".pem", # some openssl dumps have tabs ".sln", "/Makefile", "/generate_visualc_files.pl", From c3189259e7884cc5f3f8037791c167b544c5189d Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Sun, 10 May 2020 17:40:49 +0200 Subject: [PATCH 10/12] Run assemble_changelog.py in all.sh Avoid nasty surprises where it would fail when we want to make a release. Signed-off-by: Gilles Peskine --- tests/scripts/all.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 1ae331744..c33fce614 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -591,6 +591,18 @@ component_check_files () { record_status tests/scripts/check-files.py } +component_check_changelog () { + msg "Check: changelog entries" # < 1s + rm -f ChangeLog.new + record_status scripts/assemble_changelog.py -o ChangeLog.new + if [ -e ChangeLog.new ]; then + # Show the diff for information. It isn't an error if the diff is + # non-empty. + diff -u ChangeLog ChangeLog.new || true + rm ChangeLog.new + fi +} + component_check_names () { msg "test/build: declared and exported names" # < 3s record_status tests/scripts/check-names.sh -v From 14b559a4c3d3f50469a5d2a990176edfcc25259d Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 28 May 2020 18:19:20 +0200 Subject: [PATCH 11/12] Finish the documentation of normalize_path Signed-off-by: Gilles Peskine --- tests/scripts/check-files.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/scripts/check-files.py b/tests/scripts/check-files.py index 86be3ffb3..a5436fc64 100755 --- a/tests/scripts/check-files.py +++ b/tests/scripts/check-files.py @@ -46,8 +46,10 @@ class FileIssueTracker: @staticmethod def normalize_path(filepath): - """Normalize ``filepath`` """ + """Normalize ``filepath`` with / as the directory separator.""" filepath = os.path.normpath(filepath) + # On Windows, we may have backslashes to separate directories. + # We need slashes to match exemption lists. seps = os.path.sep if os.path.altsep is not None: seps += os.path.altsep From 0506f62569de084fe285c97d422e93734d16269f Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 28 May 2020 18:32:43 +0200 Subject: [PATCH 12/12] Fix an LTS version number in a changelog entry Signed-off-by: Gilles Peskine --- ChangeLog.d/bugfix.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog.d/bugfix.txt b/ChangeLog.d/bugfix.txt index b7d4cd688..e7f49d2d9 100644 --- a/ChangeLog.d/bugfix.txt +++ b/ChangeLog.d/bugfix.txt @@ -1,4 +1,4 @@ Bugfix * Fix the Visual Studio Release x64 build configuration for mbedtls itself. - Completes a previous fix in Mbed TLS 2.7 that only fixed the build for + Completes a previous fix in Mbed TLS 2.7.12 that only fixed the build for the example programs. Reported in #1430 and fix contributed by irwir.