cert_audit: Parse more information from test suite data file

Signed-off-by: Pengyu Lv <pengyu.lv@arm.com>

tests/scripts/audit-validity-dates.py

@@ -34,6 +34,9 @@ from enum import Enum
 
 from cryptography import x509
 
+# reuse the function to parse *.data file in tests/suites/
+from generate_test_code import parse_test_data as parse_suite_data
+
 class DataType(Enum):
     CRT = 1 # Certificate
     CRL = 2 # Certificate Revocation List
@@ -129,6 +132,32 @@ class X509Parser():
         else:
             return ""
 
+    @staticmethod
+    def check_hex_string(hex_str: str) -> bool:
+        """Check if the hex string is possibly DER data."""
+        hex_len = len(hex_str)
+        # At least 6 hex char for 3 bytes: Type + Length + Content
+        if hex_len < 6:
+            return False
+        # Check if Type (1 byte) is SEQUENCE.
+        if hex_str[0:2] != '30':
+            return False
+        # Check LENGTH (1 byte) value
+        content_len = int(hex_str[2:4], base=16)
+        consumed = 4
+        if content_len in (128, 255):
+            # Indefinite or Reserved
+            return False
+        elif content_len > 127:
+            # Definite, Long
+            length_len = (content_len - 128) * 2
+            content_len = int(hex_str[consumed:consumed+length_len], base=16)
+            consumed += length_len
+        # Check LENGTH
+        if hex_len != content_len * 2 + consumed:
+            return False
+        return True
+
 class Auditor:
     """A base class for audit."""
     def __init__(self, verbose):
@@ -236,6 +265,64 @@ class TestDataAuditor(Auditor):
                               for file_name in file_names)
         return data_files
 
+class FileWrapper():
+    """
+    This a stub class of generate_test_code.FileWrapper.
+
+    This class reads the whole file to memory before iterating
+    over the lines.
+    """
+
+    def __init__(self, file_name):
+        """
+        Read the file and initialize the line number to 0.
+
+        :param file_name: File path to open.
+        """
+        with open(file_name, 'rb') as f:
+            self.buf = f.read()
+        self.buf_len = len(self.buf)
+        self._line_no = 0
+        self._line_start = 0
+
+    def __iter__(self):
+        """Make the class iterable."""
+        return self
+
+    def __next__(self):
+        """
+        This method for returning a line of the file per iteration.
+
+        :return: Line read from file.
+        """
+        # If we reach the end of the file.
+        if not self._line_start < self.buf_len:
+            raise StopIteration
+
+        line_end = self.buf.find(b'\n', self._line_start) + 1
+        if line_end > 0:
+            # Find the first LF as the end of the new line.
+            line = self.buf[self._line_start:line_end]
+            self._line_start = line_end
+            self._line_no += 1
+        else:
+            # No LF found. We are at the last line without LF.
+            line = self.buf[self._line_start:]
+            self._line_start = self.buf_len
+            self._line_no += 1
+
+        # Convert byte array to string with correct encoding and
+        # strip any whitespaces added in the decoding process.
+        return line.decode(sys.getdefaultencoding()).rstrip() + '\n'
+
+    def get_line_no(self):
+        """
+        Gives current line number.
+        """
+        return self._line_no
+
+    line_no = property(get_line_no)
+
 class SuiteDataAuditor(Auditor):
     """Class for auditing files in tests/suites/*.data"""
     def __init__(self, options):
@@ -246,27 +333,31 @@ class SuiteDataAuditor(Auditor):
         """Collect all files in tests/suites/*.data"""
         test_dir = self.find_test_dir()
         suites_data_folder = os.path.join(test_dir, 'suites')
-        # collect all data files in tests/suites (114 in total)
         data_files = glob.glob(os.path.join(suites_data_folder, '*.data'))
         return data_files
 
     def parse_file(self, filename: str):
-        """Parse AuditData from file."""
+        """
-        with open(filename, 'r') as f:
+        Parse a list of AuditData from file.
-            data = f.read()
+
+        :param filename: name of the file to parse.
+        :return list of AuditData parsed from the file.
+        """
         audit_data_list = []
-        # extract hex strings from the data file.
+        data_f = FileWrapper(filename)
-        hex_strings = re.findall(r'"(?P<data>[0-9a-fA-F]+)"', data)
+        for _, _, _, test_args in parse_suite_data(data_f):
-        for hex_str in hex_strings:
+            for test_arg in test_args:
-            # We regard hex string with odd number length as invaild data.
+                match = re.match(r'"(?P<data>[0-9a-fA-F]+)"', test_arg)
-            if len(hex_str) & 1:
+                if not match:
-                continue
+                    continue
-            bytes_data = bytes.fromhex(hex_str)
+                if not X509Parser.check_hex_string(match.group('data')):
-            audit_data = self.parse_bytes(bytes_data)
+                    continue
-            if audit_data is None:
+                audit_data = self.parse_bytes(bytes.fromhex(match.group('data')))
-                continue
+                if audit_data is None:
-            audit_data.filename = filename
+                    continue
-            audit_data_list.append(audit_data)
+                audit_data.filename = filename
+                audit_data_list.append(audit_data)
+
         return audit_data_list
 
 def list_all(audit_data: AuditData):
@@ -308,9 +399,6 @@ def main():
         suite_data_files = sd_auditor.default_files
 
     td_auditor.walk_all(data_files)
-    # TODO: Improve the method for auditing test suite data files
-    #       It takes 6 times longer than td_auditor.walk_all(),
-    #       typically 0.827 s VS 0.147 s.
     sd_auditor.walk_all(suite_data_files)
 
     if args.all: