Merge remote-tracking branch 'origin/pr/2497' into mbedtls-2.16

* origin/pr/2497:
  Re-generate library/certs.c from script
  Add new line at the end of test-ca2.key.enc
  Use strict syntax to annotate origin of test data in certs.c
  Add run to all.sh exercising !MBEDTLS_PEM_PARSE_C + !MBEDTLS_FS_IO
  Allow DHM self test to run without MBEDTLS_PEM_PARSE_C
  ssl-opt.sh: Auto-skip tests that use files if MBEDTLS_FS_IO unset
  Document origin of hardcoded certificates in library/certs.c
  Adapt ChangeLog
  Rename server1.der to server1.crt.der
  Add DER encoded files to git tree
  Add build instructions to generate DER versions of CRTs and keys
  Document "none" value for ca_path/ca_file in ssl_client2/ssl_server2
  ssl_server2: Skip CA setup if `ca_path` or `ca_file` argument "none"
  ssl_client2: Skip CA setup if `ca_path` or `ca_file` argument "none"
  Correct white spaces in ssl_server2 and ssl_client2
  Adapt ssl_client2 to parse DER encoded test CRTs if PEM is disabled
  Adapt ssl_server2 to parse DER encoded test CRTs if PEM is disabled

tests/data_files/Makefile

@@ -46,14 +46,30 @@ test-ca.crt: $(test_ca_key_file_rsa) test-ca.req.sha256
 	$(MBEDTLS_CERT_WRITE) is_ca=1 serial=3 request_file=test-ca.req.sha256 selfsign=1 issuer_name="C=NL,O=PolarSSL,CN=PolarSSL Test CA" issuer_key=$(test_ca_key_file_rsa) issuer_pwd=$(test_ca_pwd_rsa) not_before=20110212144400 not_after=20210212144400 md=SHA1 version=3 output_file=$@
 all_final += test-ca.crt
 
+test-ca.crt.der: test-ca.crt
+	$(OPENSSL) x509 -inform PEM -in $< -outform DER -out $@
+all_final += test-ca.crt.der
+
+test-ca.key.der: $(test_ca_key_file_rsa)
+	$(OPENSSL) pkey -in $< -out $@ -inform PEM -outform DER -passin "pass:$(test_ca_pwd_rsa)"
+all_final += test-ca.key.der
+
 test-ca-sha1.crt: $(test_ca_key_file_rsa) test-ca.req.sha256
 	$(MBEDTLS_CERT_WRITE) is_ca=1 serial=3 request_file=test-ca.req.sha256 selfsign=1 issuer_name="C=NL,O=PolarSSL,CN=PolarSSL Test CA" issuer_key=$(test_ca_key_file_rsa) issuer_pwd=$(test_ca_pwd_rsa) not_before=20110212144400 not_after=20210212144400 md=SHA1 version=3 output_file=$@
 all_final += test-ca-sha1.crt
 
+test-ca-sha1.crt.der: test-ca-sha1.crt
+	$(OPENSSL) x509 -in $< -out $@ -inform PEM -outform DER
+all_final += test-ca-sha1.crt.der
+
 test-ca-sha256.crt: $(test_ca_key_file_rsa) test-ca.req.sha256
 	$(MBEDTLS_CERT_WRITE) is_ca=1 serial=3 request_file=test-ca.req.sha256 selfsign=1 issuer_name="C=NL,O=PolarSSL,CN=PolarSSL Test CA" issuer_key=$(test_ca_key_file_rsa) issuer_pwd=$(test_ca_pwd_rsa) not_before=20110212144400 not_after=20210212144400 md=SHA256 version=3 output_file=$@
 all_final += test-ca-sha256.crt
 
+test-ca-sha256.crt.der: test-ca-sha256.crt
+	$(OPENSSL) x509 -in $< -out $@ -inform PEM -outform DER
+all_final += test-ca-sha256.crt.der
+
 test_ca_key_file_rsa_alt = test-ca-alt.key
 
 $(test_ca_key_file_rsa_alt):
@@ -74,6 +90,14 @@ all_final += test-ca-good-alt.crt
 test_ca_crt_file_ec = test-ca2.crt
 test_ca_key_file_ec = test-ca2.key
 
+test-ca2.crt.der: $(test_ca_crt_file_ec)
+	$(OPENSSL) x509 -in $(test_ca_crt_file_ec) -out $@ -inform PEM -outform DER
+all_final += test-ca2.crt.der
+
+ test-ca2.key.der: $(test_ca_key_file_ec)
+	$(OPENSSL) pkey -in $(test_ca_key_file_ec) -out $@ -inform PEM -outform DER
+all_final += test-ca2.key.der
+
 test_ca_crt_cat12 = test-ca_cat12.crt
 $(test_ca_crt_cat12): $(test_ca_crt) $(test_ca_crt_file_ec)
 	cat $(test_ca_crt) $(test_ca_crt_file_ec) > $@
@@ -112,6 +136,14 @@ cli-rsa-sha256.crt: cli-rsa.csr
 	$(MBEDTLS_CERT_WRITE) request_file=$< serial=4 issuer_crt=$(test_ca_crt) issuer_key=$(test_ca_key_file_rsa) issuer_pwd=$(test_ca_pwd_rsa) not_before=20110212144406 not_after=20210212144406 md=SHA256 version=3 output_file=$@
 all_final += cli-rsa-sha256.crt
 
+cli-rsa-sha256.crt.der: cli-rsa-sha256.crt
+	$(OPENSSL) x509 -in $< -out $@ -inform PEM -outform DER
+all_final += cli-rsa-sha256.crt.der
+
+ cli-rsa.key.der: $(cli_crt_key_file_rsa)
+	$(OPENSSL) pkey -in $< -out $@ -inform PEM -outform DER
+all_final += cli-rsa.key.der
+
 test_ca_int_rsa1 = test-int-ca.crt
 
 server7.csr: server7.key
@@ -130,6 +162,22 @@ server7_int-ca-exp.crt: server7.crt test-int-ca-exp.crt
 	cat server7.crt test-int-ca-exp.crt > $@
 all_final += server7_int-ca-exp.crt
 
+cli2.crt.der: cli2.crt
+	$(OPENSSL) x509 -in $< -out $@ -inform PEM -outform DER
+all_final += cli2.crt.der
+
+ cli2.key.der: cli2.key
+	$(OPENSSL) pkey -in $< -out $@ -inform PEM -outform DER
+all_final += cli2.key.der
+
+ server5.crt.der: server5.crt
+	$(OPENSSL) x509 -in $< -out $@ -inform PEM -outform DER
+all_final += server5.crt.der
+
+ server5.key.der: server5.key
+	$(OPENSSL) pkey -in $< -out $@ -inform PEM -outform DER
+all_final += server5.key.der
+
 server5-ss-expired.crt: server5.key
 	$(FAKETIME) -f -3653d $(OPENSSL) req -x509 -new -subj "/C=UK/O=mbed TLS/OU=testsuite/CN=localhost" -days 3653 -sha256 -key $< -out $@
 all_final += server5-ss-expired.crt
@@ -819,6 +867,18 @@ server2.req.sha256: server2.key
 	$(MBEDTLS_CERT_REQ) output_file=$@ filename=$< subject_name="C=NL,O=PolarSSL,CN=localhost" md=SHA256
 all_intermediate += server2.req.sha256
 
+server2.crt.der: server2.crt
+	$(OPENSSL) x509 -inform PEM -in $< -outform DER -out $@
+all_final += server2.crt.der
+
+server2-sha256.crt.der: server2-sha256.crt
+	$(OPENSSL) x509 -inform PEM -in $< -outform DER -out $@
+all_final += server2-sha256.crt.der
+
+server2.key.der: server2.key
+	$(OPENSSL) pkey -in $< -out $@ -inform PEM -outform DER
+all_final += server2.key.der
+
 # server5*
 
 # The use of 'Server 1' in the DN is intentional here, as the DN is hardcoded in the x509_write test suite.'
@@ -848,9 +908,9 @@ server1.crt: server1.key server1.req.sha256 $(test_ca_crt) $(test_ca_key_file_rs
 	$(MBEDTLS_CERT_WRITE) request_file=server1.req.sha256 issuer_crt=$(test_ca_crt) issuer_key=$(test_ca_key_file_rsa) issuer_pwd=$(test_ca_pwd_rsa) version=1 not_before=20110212144406 not_after=20210212144406 md=SHA1 version=3 output_file=$@
 server1.noauthid.crt: server1.key server1.req.sha256 $(test_ca_crt) $(test_ca_key_file_rsa)
 	$(MBEDTLS_CERT_WRITE) request_file=server1.req.sha256 issuer_crt=$(test_ca_crt) issuer_key=$(test_ca_key_file_rsa) issuer_pwd=$(test_ca_pwd_rsa) not_before=20110212144406 not_after=20210212144406 md=SHA1 authority_identifier=0 version=3 output_file=$@
-server1.der: server1.crt
+server1.crt.der: server1.crt
 	$(OPENSSL) x509 -inform PEM -in $< -outform DER -out $@
-all_final += server1.crt server1.noauthid.crt server1.der
+all_final += server1.crt server1.noauthid.crt server1.crt.der
 
 server1.key_usage.crt: server1.key server1.req.sha256 $(test_ca_crt) $(test_ca_key_file_rsa)
 	$(MBEDTLS_CERT_WRITE) request_file=server1.req.sha256 issuer_crt=$(test_ca_crt) issuer_key=$(test_ca_key_file_rsa) issuer_pwd=$(test_ca_pwd_rsa) version=1 not_before=20110212144406 not_after=20210212144406 md=SHA1 key_usage=digital_signature,non_repudiation,key_encipherment version=3 output_file=$@

tests/data_files/test-ca2.key.enc

@@ -0,0 +1,9 @@
+-----BEGIN EC PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: DES-EDE3-CBC,307EAB469933D64E
+
+IxbrRmKcAzctJqPdTQLA4SWyBYYGYJVkYEna+F7Pa5t5Yg/gKADrFKcm6B72e7DG
+ihExtZI648s0zdYw6qSJ74vrPSuWDe5qm93BqsfVH9svtCzWHW0pm1p0KTBCFfUq
+UsuWTITwJImcnlAs1gaRZ3sAWm7cOUidL0fo2G0fYUFNcYoCSLffCFTEHBuPnagb
+a77x/sY1Bvii8S9/XhDTb6pTMx06wzrm
+-----END EC PRIVATE KEY-----

tests/scripts/all.sh

@@ -652,6 +652,20 @@ component_test_no_renegotiation () {
     if_build_succeeded tests/ssl-opt.sh
 }
 
+component_test_no_pem_no_fs () {
+    msg "build: Default + !MBEDTLS_PEM_PARSE_C + !MBEDTLS_FS_IO (ASan build)"
+    scripts/config.pl unset MBEDTLS_PEM_PARSE_C
+    scripts/config.pl unset MBEDTLS_FS_IO
+    CC=gcc cmake -D CMAKE_BUILD_TYPE:String=Asan .
+    make
+
+    msg "test: !MBEDTLS_PEM_PARSE_C !MBEDTLS_FS_IO - main suites (inc. selftests) (ASan build)" # ~ 50s
+    make test
+
+    msg "test: !MBEDTLS_PEM_PARSE_C !MBEDTLS_FS_IO - ssl-opt.sh (ASan build)" # ~ 6 min
+    if_build_succeeded tests/ssl-opt.sh
+}
+
 component_test_rsa_no_crt () {
     msg "build: Default + RSA_NO_CRT (ASan build)" # ~ 6 min
     scripts/config.pl set MBEDTLS_RSA_NO_CRT

tests/ssl-opt.sh

@@ -553,6 +553,20 @@ run_test() {
     CLI_EXPECT="$3"
     shift 3
 
+    # Check if test uses files
+    TEST_USES_FILES=$(echo "$SRV_CMD $CLI_CMD" | grep "\.\(key\|crt\|pem\)" )
+    if [ ! -z "$TEST_USES_FILES" ]; then
+       requires_config_enabled MBEDTLS_FS_IO
+    fi
+
+    # should we skip?
+    if [ "X$SKIP_NEXT" = "XYES" ]; then
+        SKIP_NEXT="NO"
+        echo "SKIP"
+        SKIPS=$(( $SKIPS + 1 ))
+        return
+    fi
+
     # fix client port
     if [ -n "$PXY_CMD" ]; then
         CLI_CMD=$( echo "$CLI_CMD" | sed s/+SRV_PORT/$PXY_PORT/g )

tests/suites/test_suite_x509parse.data

@@ -2,14 +2,26 @@ X509 Certificate information #1
 depends_on:MBEDTLS_PEM_PARSE_C:MBEDTLS_RSA_C:MBEDTLS_SHA1_C
 x509_cert_info:"data_files/server1.crt":"cert. version     \: 3\nserial number     \: 01\nissuer name       \: C=NL, O=PolarSSL, CN=PolarSSL Test CA\nsubject name      \: C=NL, O=PolarSSL, CN=PolarSSL Server 1\nissued  on        \: 2011-02-12 14\:44\:06\nexpires on        \: 2021-02-12 14\:44\:06\nsigned using      \: RSA with SHA1\nRSA key size      \: 2048 bits\nbasic constraints \: CA=false\n"
 
+X509 Certificate information #1 (DER)
+depends_on:MBEDTLS_RSA_C:MBEDTLS_SHA1_C
+x509_cert_info:"data_files/server1.crt.der":"cert. version     \: 3\nserial number     \: 01\nissuer name       \: C=NL, O=PolarSSL, CN=PolarSSL Test CA\nsubject name      \: C=NL, O=PolarSSL, CN=PolarSSL Server 1\nissued  on        \: 2011-02-12 14\:44\:06\nexpires on        \: 2021-02-12 14\:44\:06\nsigned using      \: RSA with SHA1\nRSA key size      \: 2048 bits\nbasic constraints \: CA=false\n"
+
 X509 Certificate information #2
 depends_on:MBEDTLS_PEM_PARSE_C:MBEDTLS_RSA_C:MBEDTLS_SHA1_C
 x509_cert_info:"data_files/server2.crt":"cert. version     \: 3\nserial number     \: 02\nissuer name       \: C=NL, O=PolarSSL, CN=PolarSSL Test CA\nsubject name      \: C=NL, O=PolarSSL, CN=localhost\nissued  on        \: 2011-02-12 14\:44\:06\nexpires on        \: 2021-02-12 14\:44\:06\nsigned using      \: RSA with SHA1\nRSA key size      \: 2048 bits\nbasic constraints \: CA=false\n"
 
+X509 Certificate information #2 (DER)
+depends_on:MBEDTLS_RSA_C:MBEDTLS_SHA1_C
+x509_cert_info:"data_files/server2.crt.der":"cert. version     \: 3\nserial number     \: 02\nissuer name       \: C=NL, O=PolarSSL, CN=PolarSSL Test CA\nsubject name      \: C=NL, O=PolarSSL, CN=localhost\nissued  on        \: 2011-02-12 14\:44\:06\nexpires on        \: 2021-02-12 14\:44\:06\nsigned using      \: RSA with SHA1\nRSA key size      \: 2048 bits\nbasic constraints \: CA=false\n"
+
 X509 Certificate information #3
 depends_on:MBEDTLS_PEM_PARSE_C:MBEDTLS_RSA_C:MBEDTLS_SHA1_C
 x509_cert_info:"data_files/test-ca.crt":"cert. version     \: 3\nserial number     \: 03\nissuer name       \: C=NL, O=PolarSSL, CN=PolarSSL Test CA\nsubject name      \: C=NL, O=PolarSSL, CN=PolarSSL Test CA\nissued  on        \: 2011-02-12 14\:44\:00\nexpires on        \: 2021-02-12 14\:44\:00\nsigned using      \: RSA with SHA1\nRSA key size      \: 2048 bits\nbasic constraints \: CA=true\n"
 
+X509 Certificate information #3 (DER)
+depends_on:MBEDTLS_RSA_C:MBEDTLS_SHA1_C
+x509_cert_info:"data_files/test-ca.crt.der":"cert. version     \: 3\nserial number     \: 03\nissuer name       \: C=NL, O=PolarSSL, CN=PolarSSL Test CA\nsubject name      \: C=NL, O=PolarSSL, CN=PolarSSL Test CA\nissued  on        \: 2011-02-12 14\:44\:00\nexpires on        \: 2021-02-12 14\:44\:00\nsigned using      \: RSA with SHA1\nRSA key size      \: 2048 bits\nbasic constraints \: CA=true\n"
+
 X509 Certificate information MD2 Digest
 depends_on:MBEDTLS_PEM_PARSE_C:MBEDTLS_RSA_C:MBEDTLS_MD2_C
 x509_cert_info:"data_files/cert_md2.crt":"cert. version     \: 3\nserial number     \: 09\nissuer name       \: C=NL, O=PolarSSL, CN=PolarSSL Test CA\nsubject name      \: C=NL, O=PolarSSL, CN=PolarSSL Cert MD2\nissued  on        \: 2009-07-12 10\:56\:59\nexpires on        \: 2011-07-12 10\:56\:59\nsigned using      \: RSA with MD2\nRSA key size      \: 2048 bits\nbasic constraints \: CA=false\n"