diff --git a/ChangeLog.d/bugfix.txt b/ChangeLog.d/bugfix.txt index 5bf0aeb72..d7ace300e 100644 --- a/ChangeLog.d/bugfix.txt +++ b/ChangeLog.d/bugfix.txt @@ -1,4 +1,4 @@ Bugfix * Fix the Visual Studio Release x64 build configuration for mbedtls itself. - Completes a previous fix in Mbed TLS 2.16 that only fixed the build for - the example programs. Reported in #1430 and fix contributed by irwir. \ No newline at end of file + Completes a previous fix in Mbed TLS 2.16.3 that only fixed the build for + the example programs. Reported in #1430 and fix contributed by irwir. diff --git a/programs/pkey/rsa_priv.txt b/programs/pkey/rsa_priv.txt index 22c37fe61..254fcf852 100644 --- a/programs/pkey/rsa_priv.txt +++ b/programs/pkey/rsa_priv.txt @@ -1,8 +1,8 @@ -N = A1D46FBA2318F8DCEF16C280948B1CF27966B9B47225ED2989F8D74B45BD36049C0AAB5AD0FF003553BA843C8E12782FC5873BB89A3DC84B883D25666CD22BF3ACD5B675969F8BEBFBCAC93FDD927C7442B178B10D1DFF9398E52316AAE0AF74E594650BDC3C670241D418684593CDA1A7B9DC4F20D2FDC6F66344074003E211 -E = 010001 -D = 589552BB4F2F023ADDDD5586D0C8FD857512D82080436678D07F984A29D892D31F1F7000FC5A39A0F73E27D885E47249A4148C8A5653EF69F91F8F736BA9F84841C2D99CD8C24DE8B72B5C9BE0EDBE23F93D731749FEA9CFB4A48DD2B7F35A2703E74AA2D4DB7DE9CEEA7D763AF0ADA7AC176C4E9A22C4CDA65CEC0C65964401 -P = CD083568D2D46C44C40C1FA0101AF2155E59C70B08423112AF0C1202514BBA5210765E29FF13036F56C7495894D80CF8C3BAEE2839BACBB0B86F6A2965F60DB1 -Q = CA0EEEA5E710E8E9811A6B846399420E3AE4A4C16647E426DDF8BBBCB11CD3F35CE2E4B6BCAD07AE2C0EC2ECBFCC601B207CDD77B5673E16382B1130BF465261 -DP = 0D0E21C07BF434B4A83B116472C2147A11D8EB98A33CFBBCF1D275EF19D815941622435AAF3839B6C432CA53CE9E772CFBE1923A937A766FD93E96E6EDEC1DF1 -DQ = 269CEBE6305DFEE4809377F078C814E37B45AE6677114DFC4F76F5097E1F3031D592567AC55B9B98213B40ECD54A4D2361F5FAACA1B1F51F71E4690893C4F081 -QP = 97AC5BB885ABCA314375E9E4DB1BA4B2218C90619F61BD474F5785075ECA81750A735199A8C191FE2D3355E7CF601A70E5CABDE0E02C2538BB9FB4871540B3C1 +N = A1D46FBA2318F8DCEF16C280948B1CF27966B9B47225ED2989F8D74B45BD36049C0AAB5AD0FF003553BA843C8E12782FC5873BB89A3DC84B883D25666CD22BF3ACD5B675969F8BEBFBCAC93FDD927C7442B178B10D1DFF9398E52316AAE0AF74E594650BDC3C670241D418684593CDA1A7B9DC4F20D2FDC6F66344074003E211 +E = 010001 +D = 589552BB4F2F023ADDDD5586D0C8FD857512D82080436678D07F984A29D892D31F1F7000FC5A39A0F73E27D885E47249A4148C8A5653EF69F91F8F736BA9F84841C2D99CD8C24DE8B72B5C9BE0EDBE23F93D731749FEA9CFB4A48DD2B7F35A2703E74AA2D4DB7DE9CEEA7D763AF0ADA7AC176C4E9A22C4CDA65CEC0C65964401 +P = CD083568D2D46C44C40C1FA0101AF2155E59C70B08423112AF0C1202514BBA5210765E29FF13036F56C7495894D80CF8C3BAEE2839BACBB0B86F6A2965F60DB1 +Q = CA0EEEA5E710E8E9811A6B846399420E3AE4A4C16647E426DDF8BBBCB11CD3F35CE2E4B6BCAD07AE2C0EC2ECBFCC601B207CDD77B5673E16382B1130BF465261 +DP = 0D0E21C07BF434B4A83B116472C2147A11D8EB98A33CFBBCF1D275EF19D815941622435AAF3839B6C432CA53CE9E772CFBE1923A937A766FD93E96E6EDEC1DF1 +DQ = 269CEBE6305DFEE4809377F078C814E37B45AE6677114DFC4F76F5097E1F3031D592567AC55B9B98213B40ECD54A4D2361F5FAACA1B1F51F71E4690893C4F081 +QP = 97AC5BB885ABCA314375E9E4DB1BA4B2218C90619F61BD474F5785075ECA81750A735199A8C191FE2D3355E7CF601A70E5CABDE0E02C2538BB9FB4871540B3C1 diff --git a/programs/pkey/rsa_pub.txt b/programs/pkey/rsa_pub.txt index 2c6d313af..1e7ae0c9c 100644 --- a/programs/pkey/rsa_pub.txt +++ b/programs/pkey/rsa_pub.txt @@ -1,2 +1,2 @@ -N = A1D46FBA2318F8DCEF16C280948B1CF27966B9B47225ED2989F8D74B45BD36049C0AAB5AD0FF003553BA843C8E12782FC5873BB89A3DC84B883D25666CD22BF3ACD5B675969F8BEBFBCAC93FDD927C7442B178B10D1DFF9398E52316AAE0AF74E594650BDC3C670241D418684593CDA1A7B9DC4F20D2FDC6F66344074003E211 -E = 010001 +N = A1D46FBA2318F8DCEF16C280948B1CF27966B9B47225ED2989F8D74B45BD36049C0AAB5AD0FF003553BA843C8E12782FC5873BB89A3DC84B883D25666CD22BF3ACD5B675969F8BEBFBCAC93FDD927C7442B178B10D1DFF9398E52316AAE0AF74E594650BDC3C670241D418684593CDA1A7B9DC4F20D2FDC6F66344074003E211 +E = 010001 diff --git a/tests/data_files/bitstring-in-dn.pem b/tests/data_files/bitstring-in-dn.pem index 1a98aa3ac..c50bd6684 100644 --- a/tests/data_files/bitstring-in-dn.pem +++ b/tests/data_files/bitstring-in-dn.pem @@ -1,51 +1,51 @@ ------BEGIN CERTIFICATE----- -MIIEATCCAumgAwIBAgIBAjANBgkqhkiG9w0BAQUFADBxMRMwEQYDVQQDDApUZXN0 -IENBIDAxMREwDwYDVQQIDAhFY25pdm9ycDELMAkGA1UEBhMCWFgxHjAcBgkqhkiG -9w0BCQEWD3RjYUBleGFtcGxlLmNvbTEaMBgGA1UECgwRVGVzdCBDQSBBdXRob3Jp -dHkwHhcNMTUwMzExMTIwNjUxWhcNMjUwMzA4MTIwNjUxWjCBmzELMAkGA1UEBhMC -WFgxDDAKBgNVBAoMA3RjYTERMA8GA1UECAwIRWNuaXZvcnAxDDAKBgNVBAsMA1RD -QTEPMA0GA1UEAwwGQ2xpZW50MSEwHwYJKoZIhvcNAQkBFhJjbGllbnRAZXhhbXBs -ZS5jb20xEzARBgNVBAUTCjcxMDEwMTIyNTUxFDASBgNVBC0DCwA3MTAxMDEyMjU1 -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnQS0JLb8Dqy8V2mszkWk -V8c/NPQcG3ivueXZHqOT9JTiPqrigGcLHtlmlaJ0aUUxix7q60aOds041TFyeknT -SUFYY4ppOhiP+fOpWKPv4ZMwhSI2XzcgYhQSNHV0lIG1we9RAAfumomDMq7oMJhb -EGf0ihibbwZXPUwBlm10GaB4K93PNY8Bz4ekBxzQ1WJkQ5LGsQnVZSuLnvp5dWSe -J2axxyY4hPXR30jzEyZvy4kv4nzAu5lqZ5XKLrRO4TKwZrtr+CCPVkPJRE36rWYt -tQaJEEpNOo0ZPpTtG6F8/tGh5r8jFx/f6wG+nyANJJ98kEP8i6TPjRrg+697mLcd -iQIDAQABo3kwdzAJBgNVHRMEAjAAMDYGA1UdHwQvMC0wK6ApoCeGJWh0dHA6Ly9j -cmwuZXhhbXBsZS5jb20vdGVzdF9jYV8wMS5jcmwwEwYDVR0lBAwwCgYIKwYBBQUH -AwIwHQYDVR0RBBYwFIESY2xpZW50QGV4YW1wbGUuY29tMA0GCSqGSIb3DQEBBQUA -A4IBAQBySELCnU8/PtGIG3dwhJENOSU5R7w8jpRXxHCuSBR+W6nuUCISz+z+EdF/ -A7AOJDASuS+4gkrSSmQhGFpf7E5VbF8trVZhLAZrXqKMcUreKH6v0I8MAUXmIs3G -tqiBGf7pSYJN9DvVOOgANjdy6THuUzYv5qSvBZ4pNYEfHSlMNrV7niynd8dgPOML -pA7GUfv5k2mMkMbSD15pTMgcavrBKYgyqcvF1C3qghfoL5+i38H8sKzF8hy7wHtE -ESHtBq20RYA3m0UcA0e64GcanO2Ps/AQVBc7qMeHbqnqj3uUhtTkQcMUWnMgy1NR -5RbzoLMOxq7hoOCyIaQeM/wgxeGE ------END CERTIFICATE----- ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAnQS0JLb8Dqy8V2mszkWkV8c/NPQcG3ivueXZHqOT9JTiPqri -gGcLHtlmlaJ0aUUxix7q60aOds041TFyeknTSUFYY4ppOhiP+fOpWKPv4ZMwhSI2 -XzcgYhQSNHV0lIG1we9RAAfumomDMq7oMJhbEGf0ihibbwZXPUwBlm10GaB4K93P -NY8Bz4ekBxzQ1WJkQ5LGsQnVZSuLnvp5dWSeJ2axxyY4hPXR30jzEyZvy4kv4nzA -u5lqZ5XKLrRO4TKwZrtr+CCPVkPJRE36rWYttQaJEEpNOo0ZPpTtG6F8/tGh5r8j -Fx/f6wG+nyANJJ98kEP8i6TPjRrg+697mLcdiQIDAQABAoIBAF7i3MnjGmbz080v -OxJb23iAG54wdlvTjr3UPGTbjSmcXyxnsADQRFQcJHYAekCzY8EiqewL80OvuMx8 -2SU1P81hA70Dg5tsBHWT3Z6HUwsKG6QYjKr1cUhTwLyazhyAVgogSN6v7GzO9M3I -DOBw8Xb0mz5oqGVre4S7TapN8n8ZG5oWm0XKGACXy0KbzY0KvWdkUzumFQ8X/ARE -FsWyu+O69EbMqZRUKu45SrcubsdVGjOwseZHkmp5V6pc6Q/OrTHZqXJtDva5UIRq -+Lof5scy9jiwwRnM/klvh23mz0ySU4YA3645m5KqyWR4YJCR1MnMANmXUSeYWfYz -19+R1gECgYEAzm83lI7eIhTH38H0/jFpf3R7vNjPX3TR5waa4EXsCxhTOpoL89mR -iNmzH0aOC4OR8rz/9PCnwmtH1lyQ4r/RokBmCp3pBxeWSlenFfV3rLCeEDo0Q/OL -SX5DL4IbZD0VmNDt606WS7AEv93GhpN03Anw6kgHQUm1l030PR9DYZECgYEAwrgO -/RyB/Ehw7smlysZb2sn1lvd6z8fg+pcu8ZNRKODaYCCOb8p1lnHrnIQdEmjhlmVp -HAEuJ5jxCb+lyruV+dlx+0W/p6lHtKr0iBHG8EFkHnjN6Y+59Qu0HfSm0pZw7Ftr -QcUDPuDJkTVUAvrZqciWlwzTWCC9KYXtasT+AHkCgYEAnP80dAUbpyvKvr/RxShr -JYW/PWZegChmIp+BViOXWvDLC3xwrqm+5yc59QVBrjwH2WYn+26zB0dzwPFxNyHP -GuiDMnvZ54zmve9foXGn7Gv+KjU53pvwSJqAGjeHAXr7W9c5uoVwBGv/kLPn8h1e -+KGO2X6iFeMq+cFNiNan9iECgYBj+oGnsKWFVeS2ls8LyMGNGzmAZF2opiZ8RHgU -DeIULS+zP8Qi3j92GdQyLxuGQlfiEvvfJzP9nOfWa5LC/4JIIUAHFo8LlT1+JHEe -FJKi9dBkXP7NN8DxcyruXpnxctFUarQttuytslmMt2cFiKuOI7I+qJUzoMu/sEZx -FeidcQKBgQCuralmtbl4nxjn3aR/ZgFTAKCL9WaJPh5hFJ9q4UuWxJdBX5z3Ey3/ -70ehLKYPumjmZtXynzz4BTWj1W9X+tgj/499uzV6LdQERGjT6WVy8xR9RELWW0an -N9N1IAc4qTSjbI4EIMwMBSAoFfCux/jfDkG4g+RDnpV92sqxz2CtKg== ------END RSA PRIVATE KEY----- \ No newline at end of file +-----BEGIN CERTIFICATE----- +MIIEATCCAumgAwIBAgIBAjANBgkqhkiG9w0BAQUFADBxMRMwEQYDVQQDDApUZXN0 +IENBIDAxMREwDwYDVQQIDAhFY25pdm9ycDELMAkGA1UEBhMCWFgxHjAcBgkqhkiG +9w0BCQEWD3RjYUBleGFtcGxlLmNvbTEaMBgGA1UECgwRVGVzdCBDQSBBdXRob3Jp +dHkwHhcNMTUwMzExMTIwNjUxWhcNMjUwMzA4MTIwNjUxWjCBmzELMAkGA1UEBhMC +WFgxDDAKBgNVBAoMA3RjYTERMA8GA1UECAwIRWNuaXZvcnAxDDAKBgNVBAsMA1RD +QTEPMA0GA1UEAwwGQ2xpZW50MSEwHwYJKoZIhvcNAQkBFhJjbGllbnRAZXhhbXBs +ZS5jb20xEzARBgNVBAUTCjcxMDEwMTIyNTUxFDASBgNVBC0DCwA3MTAxMDEyMjU1 +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnQS0JLb8Dqy8V2mszkWk +V8c/NPQcG3ivueXZHqOT9JTiPqrigGcLHtlmlaJ0aUUxix7q60aOds041TFyeknT +SUFYY4ppOhiP+fOpWKPv4ZMwhSI2XzcgYhQSNHV0lIG1we9RAAfumomDMq7oMJhb +EGf0ihibbwZXPUwBlm10GaB4K93PNY8Bz4ekBxzQ1WJkQ5LGsQnVZSuLnvp5dWSe +J2axxyY4hPXR30jzEyZvy4kv4nzAu5lqZ5XKLrRO4TKwZrtr+CCPVkPJRE36rWYt +tQaJEEpNOo0ZPpTtG6F8/tGh5r8jFx/f6wG+nyANJJ98kEP8i6TPjRrg+697mLcd +iQIDAQABo3kwdzAJBgNVHRMEAjAAMDYGA1UdHwQvMC0wK6ApoCeGJWh0dHA6Ly9j +cmwuZXhhbXBsZS5jb20vdGVzdF9jYV8wMS5jcmwwEwYDVR0lBAwwCgYIKwYBBQUH +AwIwHQYDVR0RBBYwFIESY2xpZW50QGV4YW1wbGUuY29tMA0GCSqGSIb3DQEBBQUA +A4IBAQBySELCnU8/PtGIG3dwhJENOSU5R7w8jpRXxHCuSBR+W6nuUCISz+z+EdF/ +A7AOJDASuS+4gkrSSmQhGFpf7E5VbF8trVZhLAZrXqKMcUreKH6v0I8MAUXmIs3G +tqiBGf7pSYJN9DvVOOgANjdy6THuUzYv5qSvBZ4pNYEfHSlMNrV7niynd8dgPOML +pA7GUfv5k2mMkMbSD15pTMgcavrBKYgyqcvF1C3qghfoL5+i38H8sKzF8hy7wHtE +ESHtBq20RYA3m0UcA0e64GcanO2Ps/AQVBc7qMeHbqnqj3uUhtTkQcMUWnMgy1NR +5RbzoLMOxq7hoOCyIaQeM/wgxeGE +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAnQS0JLb8Dqy8V2mszkWkV8c/NPQcG3ivueXZHqOT9JTiPqri +gGcLHtlmlaJ0aUUxix7q60aOds041TFyeknTSUFYY4ppOhiP+fOpWKPv4ZMwhSI2 +XzcgYhQSNHV0lIG1we9RAAfumomDMq7oMJhbEGf0ihibbwZXPUwBlm10GaB4K93P +NY8Bz4ekBxzQ1WJkQ5LGsQnVZSuLnvp5dWSeJ2axxyY4hPXR30jzEyZvy4kv4nzA +u5lqZ5XKLrRO4TKwZrtr+CCPVkPJRE36rWYttQaJEEpNOo0ZPpTtG6F8/tGh5r8j +Fx/f6wG+nyANJJ98kEP8i6TPjRrg+697mLcdiQIDAQABAoIBAF7i3MnjGmbz080v +OxJb23iAG54wdlvTjr3UPGTbjSmcXyxnsADQRFQcJHYAekCzY8EiqewL80OvuMx8 +2SU1P81hA70Dg5tsBHWT3Z6HUwsKG6QYjKr1cUhTwLyazhyAVgogSN6v7GzO9M3I +DOBw8Xb0mz5oqGVre4S7TapN8n8ZG5oWm0XKGACXy0KbzY0KvWdkUzumFQ8X/ARE +FsWyu+O69EbMqZRUKu45SrcubsdVGjOwseZHkmp5V6pc6Q/OrTHZqXJtDva5UIRq ++Lof5scy9jiwwRnM/klvh23mz0ySU4YA3645m5KqyWR4YJCR1MnMANmXUSeYWfYz +19+R1gECgYEAzm83lI7eIhTH38H0/jFpf3R7vNjPX3TR5waa4EXsCxhTOpoL89mR +iNmzH0aOC4OR8rz/9PCnwmtH1lyQ4r/RokBmCp3pBxeWSlenFfV3rLCeEDo0Q/OL +SX5DL4IbZD0VmNDt606WS7AEv93GhpN03Anw6kgHQUm1l030PR9DYZECgYEAwrgO +/RyB/Ehw7smlysZb2sn1lvd6z8fg+pcu8ZNRKODaYCCOb8p1lnHrnIQdEmjhlmVp +HAEuJ5jxCb+lyruV+dlx+0W/p6lHtKr0iBHG8EFkHnjN6Y+59Qu0HfSm0pZw7Ftr +QcUDPuDJkTVUAvrZqciWlwzTWCC9KYXtasT+AHkCgYEAnP80dAUbpyvKvr/RxShr +JYW/PWZegChmIp+BViOXWvDLC3xwrqm+5yc59QVBrjwH2WYn+26zB0dzwPFxNyHP +GuiDMnvZ54zmve9foXGn7Gv+KjU53pvwSJqAGjeHAXr7W9c5uoVwBGv/kLPn8h1e ++KGO2X6iFeMq+cFNiNan9iECgYBj+oGnsKWFVeS2ls8LyMGNGzmAZF2opiZ8RHgU +DeIULS+zP8Qi3j92GdQyLxuGQlfiEvvfJzP9nOfWa5LC/4JIIUAHFo8LlT1+JHEe +FJKi9dBkXP7NN8DxcyruXpnxctFUarQttuytslmMt2cFiKuOI7I+qJUzoMu/sEZx +FeidcQKBgQCuralmtbl4nxjn3aR/ZgFTAKCL9WaJPh5hFJ9q4UuWxJdBX5z3Ey3/ +70ehLKYPumjmZtXynzz4BTWj1W9X+tgj/499uzV6LdQERGjT6WVy8xR9RELWW0an +N9N1IAc4qTSjbI4EIMwMBSAoFfCux/jfDkG4g+RDnpV92sqxz2CtKg== +-----END RSA PRIVATE KEY----- diff --git a/tests/data_files/test-ca.server1.opensslconf b/tests/data_files/test-ca.server1.opensslconf index 4a5072eae..209b0fffa 100644 --- a/tests/data_files/test-ca.server1.opensslconf +++ b/tests/data_files/test-ca.server1.opensslconf @@ -1,6 +1,6 @@ [ ca ] default_ca = test-ca - + [ test-ca ] certificate = test-ca.crt private_key = test-ca.key diff --git a/tests/scripts/all.sh b/tests/scripts/all.sh index 62156da60..9c46e2c57 100755 --- a/tests/scripts/all.sh +++ b/tests/scripts/all.sh @@ -589,6 +589,18 @@ component_check_files () { record_status tests/scripts/check-files.py } +component_check_changelog () { + msg "Check: changelog entries" # < 1s + rm -f ChangeLog.new + record_status scripts/assemble_changelog.py -o ChangeLog.new + if [ -e ChangeLog.new ]; then + # Show the diff for information. It isn't an error if the diff is + # non-empty. + diff -u ChangeLog ChangeLog.new || true + rm ChangeLog.new + fi +} + component_check_names () { msg "test/build: declared and exported names" # < 3s record_status tests/scripts/check-names.sh -v diff --git a/tests/scripts/check-files.py b/tests/scripts/check-files.py index cd993e260..a5436fc64 100755 --- a/tests/scripts/check-files.py +++ b/tests/scripts/check-files.py @@ -14,6 +14,8 @@ import os import argparse import logging import codecs +import re +import subprocess import sys @@ -23,28 +25,48 @@ class FileIssueTracker: To implement a checker that processes a file as a whole, inherit from this class and implement `check_file_for_issue` and define ``heading``. - ``files_exemptions``: files whose name ends with a string in this set + ``suffix_exemptions``: files whose name ends with a string in this set will not be checked. + ``path_exemptions``: files whose path (relative to the root of the source + tree) matches this regular expression will not be checked. This can be + ``None`` to match no path. Paths are normalized and converted to ``/`` + separators before matching. + ``heading``: human-readable description of the issue """ - files_exemptions = frozenset() + suffix_exemptions = frozenset() + path_exemptions = None # heading must be defined in derived classes. # pylint: disable=no-member def __init__(self): self.files_with_issues = {} + @staticmethod + def normalize_path(filepath): + """Normalize ``filepath`` with / as the directory separator.""" + filepath = os.path.normpath(filepath) + # On Windows, we may have backslashes to separate directories. + # We need slashes to match exemption lists. + seps = os.path.sep + if os.path.altsep is not None: + seps += os.path.altsep + return '/'.join(filepath.split(seps)) + def should_check_file(self, filepath): """Whether the given file name should be checked. - Files whose name ends with a string listed in ``self.files_exemptions`` - will not be checked. + Files whose name ends with a string listed in ``self.suffix_exemptions`` + or whose path matches ``self.path_exemptions`` will not be checked. """ - for files_exemption in self.files_exemptions: + for files_exemption in self.suffix_exemptions: if filepath.endswith(files_exemption): return False + if self.path_exemptions and \ + re.match(self.path_exemptions, self.normalize_path(filepath)): + return False return True def check_file_for_issue(self, filepath): @@ -73,6 +95,17 @@ class FileIssueTracker: logger.info(filename) logger.info("") +BINARY_FILE_PATH_RE_LIST = [ + r'docs/.*\.pdf\Z', + r'programs/fuzz/corpuses/[^.]+\Z', + r'tests/data_files/[^.]+\Z', + r'tests/data_files/.*\.(crt|csr|db|der|key|pubkey)\Z', + r'tests/data_files/.*\.req\.[^/]+\Z', + r'tests/data_files/.*malformed[^/]+\Z', + r'tests/data_files/format_pkcs12\.fmt\Z', +] +BINARY_FILE_PATH_RE = re.compile('|'.join(BINARY_FILE_PATH_RE_LIST)) + class LineIssueTracker(FileIssueTracker): """Base class for line-by-line issue tracking. @@ -80,6 +113,9 @@ class LineIssueTracker(FileIssueTracker): this class and implement `line_with_issue`. """ + # Exclude binary files. + path_exemptions = BINARY_FILE_PATH_RE + def issue_with_line(self, line, filepath): """Check the specified line for the issue that this class is for. @@ -103,7 +139,7 @@ class LineIssueTracker(FileIssueTracker): def is_windows_file(filepath): _root, ext = os.path.splitext(filepath) - return ext in ('.bat', '.dsp', '.sln', '.vcxproj') + return ext in ('.bat', '.dsp', '.dsw', '.sln', '.vcxproj') class PermissionIssueTracker(FileIssueTracker): @@ -126,9 +162,18 @@ class EndOfFileNewlineIssueTracker(FileIssueTracker): heading = "Missing newline at end of file:" + path_exemptions = BINARY_FILE_PATH_RE + def check_file_for_issue(self, filepath): with open(filepath, "rb") as f: - if not f.read().endswith(b"\n"): + try: + f.seek(-1, 2) + except OSError: + # This script only works on regular files. If we can't seek + # 1 before the end, it means that this position is before + # the beginning of the file, i.e. that the file is empty. + return + if f.read(1) != b"\n": self.files_with_issues[filepath] = None @@ -138,7 +183,8 @@ class Utf8BomIssueTracker(FileIssueTracker): heading = "UTF-8 BOM present:" - files_exemptions = frozenset([".vcxproj", ".sln"]) + suffix_exemptions = frozenset([".vcxproj", ".sln"]) + path_exemptions = BINARY_FILE_PATH_RE def check_file_for_issue(self, filepath): with open(filepath, "rb") as f: @@ -152,6 +198,8 @@ class UnixLineEndingIssueTracker(LineIssueTracker): heading = "Non-Unix line endings:" def should_check_file(self, filepath): + if not super().should_check_file(filepath): + return False return not is_windows_file(filepath) def issue_with_line(self, line, _filepath): @@ -164,6 +212,8 @@ class WindowsLineEndingIssueTracker(LineIssueTracker): heading = "Non-Windows line endings:" def should_check_file(self, filepath): + if not super().should_check_file(filepath): + return False return is_windows_file(filepath) def issue_with_line(self, line, _filepath): @@ -174,7 +224,7 @@ class TrailingWhitespaceIssueTracker(LineIssueTracker): """Track lines with trailing whitespace.""" heading = "Trailing whitespace:" - files_exemptions = frozenset([".dsp", ".md"]) + suffix_exemptions = frozenset([".dsp", ".md"]) def issue_with_line(self, line, _filepath): return line.rstrip(b"\r\n") != line.rstrip() @@ -184,7 +234,8 @@ class TabIssueTracker(LineIssueTracker): """Track lines with tabs.""" heading = "Tabs present:" - files_exemptions = frozenset([ + suffix_exemptions = frozenset([ + ".pem", # some openssl dumps have tabs ".sln", "/Makefile", "/generate_visualc_files.pl", @@ -222,31 +273,6 @@ class IntegrityChecker: self.check_repo_path() self.logger = None self.setup_logger(log_file) - self.extensions_to_check = ( - ".bat", - ".c", - ".data", - ".dsp", - ".function", - ".h", - ".md", - ".pl", - ".py", - ".sh", - ".sln", - ".vcxproj", - "/CMakeLists.txt", - "/ChangeLog", - "/Makefile", - ) - self.excluded_directories = [ - '.git', - 'mbed-os', - ] - self.excluded_paths = list(map(os.path.normpath, [ - 'cov-int', - 'examples', - ])) self.issues_to_check = [ PermissionIssueTracker(), EndOfFileNewlineIssueTracker(), @@ -273,23 +299,22 @@ class IntegrityChecker: console = logging.StreamHandler() self.logger.addHandler(console) - def prune_branch(self, root, d): - if d in self.excluded_directories: - return True - if os.path.normpath(os.path.join(root, d)) in self.excluded_paths: - return True - return False + @staticmethod + def collect_files(): + bytes_output = subprocess.check_output(['git', 'ls-files', '-z']) + bytes_filepaths = bytes_output.split(b'\0')[:-1] + ascii_filepaths = map(lambda fp: fp.decode('ascii'), bytes_filepaths) + # Prepend './' to files in the top-level directory so that + # something like `'/Makefile' in fp` matches in the top-level + # directory as well as in subdirectories. + return [fp if os.path.dirname(fp) else os.path.join(os.curdir, fp) + for fp in ascii_filepaths] def check_files(self): - for root, dirs, files in os.walk("."): - dirs[:] = sorted(d for d in dirs if not self.prune_branch(root, d)) - for filename in sorted(files): - filepath = os.path.join(root, filename) - if not filepath.endswith(self.extensions_to_check): - continue - for issue_to_check in self.issues_to_check: - if issue_to_check.should_check_file(filepath): - issue_to_check.check_file_for_issue(filepath) + for issue_to_check in self.issues_to_check: + for filepath in self.collect_files(): + if issue_to_check.should_check_file(filepath): + issue_to_check.check_file_for_issue(filepath) def output_issues(self): integrity_return_code = 0