eden-emu/mbedtls
15
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Improve code style consistency in check_names.py

Browse source 
Signed-off-by: Yuto Takano <yuto.takano@arm.com>
This commit is contained in:
Yuto Takano 2021-08-09 12:45:51 +01:00
parent 68d241211b
commit d70d446d69
1 changed files with 133 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

236
tests/scripts/check_names.py
View file

@ -81,11 +81,9 @@ class Match(): # pylint: disable=too-few-public-methods
class Problem(): # pylint: disable=too-few-public-methods class Problem(): # pylint: disable=too-few-public-methods
""" """
A parent class representing a form of static analysis error. A parent class representing a form of static analysis error.
Fields:
* textwrapper: a TextWrapper instance to format problems nicely.
""" """
def __init__(self): def __init__(self):
self.quiet = False
self.textwrapper = textwrap.TextWrapper() self.textwrapper = textwrap.TextWrapper()
self.textwrapper.width = 80 self.textwrapper.width = 80
self.textwrapper.initial_indent = " > " self.textwrapper.initial_indent = " > "
@ -100,9 +98,8 @@ class SymbolNotInHeader(Problem): # pylint: disable=too-few-public-methods
Fields: Fields:
* symbol_name: the name of the symbol. * symbol_name: the name of the symbol.
""" """
def __init__(self, symbol_name, quiet=False): def __init__(self, symbol_name):
self.symbol_name = symbol_name self.symbol_name = symbol_name
self.quiet = quiet
Problem.__init__(self) Problem.__init__(self)
def __str__(self): def __str__(self):
@ -123,19 +120,17 @@ class PatternMismatch(Problem): # pylint: disable=too-few-public-methods
* pattern: the expected regex pattern * pattern: the expected regex pattern
* match: the Match object in question * match: the Match object in question
""" """
def __init__(self, pattern, match, quiet=False): def __init__(self, pattern, match):
self.pattern = pattern self.pattern = pattern
self.match = match self.match = match
self.quiet = quiet
Problem.__init__(self) Problem.__init__(self)
def __str__(self): def __str__(self):
if self.quiet: if self.quiet:
return ("{0}:{1}:{3}" return (
.format( "{0}:{1}:{3}"
self.match.filename, .format(self.match.filename, self.match.pos[0], self.match.name)
self.match.pos[0], )
self.match.name))
return self.textwrapper.fill( return self.textwrapper.fill(
"{0}:{1}: '{2}' does not match the required pattern '{3}'." "{0}:{1}: '{2}' does not match the required pattern '{3}'."
@ -143,7 +138,9 @@ class PatternMismatch(Problem): # pylint: disable=too-few-public-methods
self.match.filename, self.match.filename,
self.match.pos[0], self.match.pos[0],
self.match.name, self.match.name,
self.pattern)) + "\n" + str(self.match) self.pattern
)
) + "\n" + str(self.match)
class Typo(Problem): # pylint: disable=too-few-public-methods class Typo(Problem): # pylint: disable=too-few-public-methods
""" """
@ -153,27 +150,23 @@ class Typo(Problem): # pylint: disable=too-few-public-methods
Fields: Fields:
* match: the Match object of the MBED name in question. * match: the Match object of the MBED name in question.
""" """
def __init__(self, match, quiet=False): def __init__(self, match):
self.match = match self.match = match
self.quiet = quiet
Problem.__init__(self) Problem.__init__(self)
def __str__(self): def __str__(self):
if self.quiet: if self.quiet:
return ("{0}:{1}:{2}" return (
.format( "{0}:{1}:{2}"
self.match.filename, .format(self.match.filename, self.match.pos[0], self.match.name)
self.match.pos[0], )
self.match.name))
return self.textwrapper.fill( return self.textwrapper.fill(
"{0}:{1}: '{2}' looks like a typo. It was not found in any " "{0}:{1}: '{2}' looks like a typo. It was not found in any "
"macros or any enums. If this is not a typo, put " "macros or any enums. If this is not a typo, put "
"//no-check-names after it." "//no-check-names after it."
.format( .format(self.match.filename, self.match.pos[0], self.match.name)
self.match.filename, ) + "\n" + str(self.match)
self.match.pos[0],
self.match.name)) + "\n" + str(self.match)
class NameCheck(): class NameCheck():
""" """
@ -184,7 +177,6 @@ class NameCheck():
self.log = None self.log = None
self.check_repo_path() self.check_repo_path()
self.return_code = 0 self.return_code = 0
self.setup_logger(verbose) self.setup_logger(verbose)
# Globally excluded filenames # Globally excluded filenames
@ -193,11 +185,6 @@ class NameCheck():
# Will contain the parse result after a comprehensive parse # Will contain the parse result after a comprehensive parse
self.parse_result = {} self.parse_result = {}
def set_return_code(self, return_code):
if return_code > self.return_code:
self.log.debug("Setting new return code to {}".format(return_code))
self.return_code = return_code
@staticmethod @staticmethod
def check_repo_path(): def check_repo_path():
""" """
@ -207,6 +194,11 @@ class NameCheck():
if not all(os.path.isdir(d) for d in ["include", "library", "tests"]): if not all(os.path.isdir(d) for d in ["include", "library", "tests"]):
raise Exception("This script must be run from Mbed TLS root") raise Exception("This script must be run from Mbed TLS root")
def set_return_code(self, return_code):
if return_code > self.return_code:
self.log.debug("Setting new return code to {}".format(return_code))
self.return_code = return_code
def setup_logger(self, verbose=False): def setup_logger(self, verbose=False):
""" """
Set up a logger and set the change the default logging level from Set up a logger and set the change the default logging level from
@ -247,28 +239,35 @@ class NameCheck():
""" """
self.log.info("Parsing source code...") self.log.info("Parsing source code...")
self.log.debug( self.log.debug(
"The following files are excluded from the search: {}" "The following filenames are excluded from the search: {}"
.format(str(self.excluded_files)) .format(str(self.excluded_files))
) )
m_headers = self.get_files("include/mbedtls/*.h") m_headers = self.get_files("include/mbedtls/*.h")
p_headers = self.get_files("include/psa/*.h") p_headers = self.get_files("include/psa/*.h")
t_headers = ["3rdparty/everest/include/everest/everest.h", t_headers = [
"3rdparty/everest/include/everest/x25519.h"] "3rdparty/everest/include/everest/everest.h",
"3rdparty/everest/include/everest/x25519.h"
]
d_headers = self.get_files("tests/include/test/drivers/*.h") d_headers = self.get_files("tests/include/test/drivers/*.h")
l_headers = self.get_files("library/*.h") l_headers = self.get_files("library/*.h")
libraries = self.get_files("library/*.c") + [ libraries = self.get_files("library/*.c") + [
"3rdparty/everest/library/everest.c", "3rdparty/everest/library/everest.c",
"3rdparty/everest/library/x25519.c"] "3rdparty/everest/library/x25519.c"
]
all_macros = self.parse_macros( all_macros = self.parse_macros(
m_headers + p_headers + t_headers + l_headers + d_headers) m_headers + p_headers + t_headers + l_headers + d_headers
)
enum_consts = self.parse_enum_consts( enum_consts = self.parse_enum_consts(
m_headers + l_headers + t_headers) m_headers + l_headers + t_headers
)
identifiers = self.parse_identifiers( identifiers = self.parse_identifiers(
m_headers + p_headers + t_headers + l_headers) m_headers + p_headers + t_headers + l_headers
)
mbed_words = self.parse_mbed_words( mbed_words = self.parse_mbed_words(
m_headers + p_headers + t_headers + l_headers + libraries) m_headers + p_headers + t_headers + l_headers + libraries
)
symbols = self.parse_symbols() symbols = self.parse_symbols()
# Remove identifier macros like mbedtls_printf or mbedtls_calloc # Remove identifier macros like mbedtls_printf or mbedtls_calloc
@ -279,7 +278,7 @@ class NameCheck():
actual_macros.append(macro) actual_macros.append(macro)
self.log.debug("Found:") self.log.debug("Found:")
self.log.debug(" {} Macros".format(len(all_macros))) self.log.debug(" {} Total Macros".format(len(all_macros)))
self.log.debug(" {} Non-identifier Macros".format(len(actual_macros))) self.log.debug(" {} Non-identifier Macros".format(len(actual_macros)))
self.log.debug(" {} Enum Constants".format(len(enum_consts))) self.log.debug(" {} Enum Constants".format(len(enum_consts)))
self.log.debug(" {} Identifiers".format(len(identifiers))) self.log.debug(" {} Identifiers".format(len(identifiers)))
@ -294,12 +293,12 @@ class NameCheck():
"mbed_words": mbed_words "mbed_words": mbed_words
} }
def parse_macros(self, header_files): def parse_macros(self, files):
""" """
Parse all macros defined by #define preprocessor directives. Parse all macros defined by #define preprocessor directives.
Args: Args:
* header_files: A List of filepaths to look through. * files: A List of filepaths to look through.
Returns a List of Match objects for the found macros. Returns a List of Match objects for the found macros.
""" """
@ -308,20 +307,22 @@ class NameCheck():
"asm", "inline", "EMIT", "_CRT_SECURE_NO_DEPRECATE", "MULADDC_" "asm", "inline", "EMIT", "_CRT_SECURE_NO_DEPRECATE", "MULADDC_"
) )
self.log.debug("Looking for macros in {} files".format(len(header_files))) self.log.debug("Looking for macros in {} files".format(len(files)))
macros = [] macros = []
for header_file in header_files: for header_file in files:
with open(header_file, "r", encoding="utf-8") as header: with open(header_file, "r", encoding="utf-8") as header:
for line_no, line in enumerate(header): for line_no, line in enumerate(header):
for macro in macro_regex.finditer(line): for macro in macro_regex.finditer(line):
if not macro.group("macro").startswith(exclusions): if macro.group("macro").startswith(exclusions):
macros.append(Match( continue
header_file,
line, macros.append(Match(
(line_no, macro.start(), macro.end()), header_file,
macro.group("macro"))) line,
(line_no, macro.start(), macro.end()),
macro.group("macro")))
return macros return macros
@ -359,20 +360,23 @@ class NameCheck():
return mbed_words return mbed_words
def parse_enum_consts(self, header_files): def parse_enum_consts(self, files):
""" """
Parse all enum value constants that are declared. Parse all enum value constants that are declared.
Args: Args:
* header_files: A List of filepaths to look through. * files: A List of filepaths to look through.
Returns a List of Match objects for the findings. Returns a List of Match objects for the findings.
""" """
self.log.debug("Looking for enum consts in {} files".format(len(header_files))) self.log.debug(
"Looking for enum consts in {} files"
.format(len(files))
)
enum_consts = [] enum_consts = []
for header_file in header_files: for header_file in files:
# Emulate a finite state machine to parse enum declarations. # Emulate a finite state machine to parse enum declarations.
# 0 = not in enum # 0 = not in enum
# 1 = inside enum # 1 = inside enum
@ -393,22 +397,26 @@ class NameCheck():
state = 0 state = 0
elif state == 1 and not re.match(r" *#", line): elif state == 1 and not re.match(r" *#", line):
enum_const = re.match(r" *(?P<enum_const>\w+)", line) enum_const = re.match(r" *(?P<enum_const>\w+)", line)
if enum_const: if not enum_const:
enum_consts.append(Match( continue
header_file,
line, enum_consts.append(Match(
(line_no, enum_const.start(), enum_const.end()), header_file,
enum_const.group("enum_const"))) line,
(line_no, enum_const.start(), enum_const.end()),
enum_const.group("enum_const")))
return enum_consts return enum_consts
def parse_identifiers(self, header_files): def parse_identifiers(self, files):
""" """
Parse all lines of a header where a function identifier is declared, Parse all lines of a header where a function identifier is declared,
based on some huersitics. Highly dependent on formatting style. based on some huersitics. Highly dependent on formatting style.
Note: .match() checks at the beginning of the string (implicit ^), while
.search() checks throughout.
Args: Args:
* header_files: A List of filepaths to look through. * files: A List of filepaths to look through.
Returns a List of Match objects with identifiers. Returns a List of Match objects with identifiers.
""" """
@ -425,23 +433,31 @@ class NameCheck():
# Match names of named data structures. # Match names of named data structures.
r"(?:typedef +)?(?:struct|union|enum) +(\w+)(?: *{)?$|" r"(?:typedef +)?(?:struct|union|enum) +(\w+)(?: *{)?$|"
# Match names of typedef instances, after closing bracket. # Match names of typedef instances, after closing bracket.
r"}? *(\w+)[;[].*") r"}? *(\w+)[;[].*"
exclusion_lines = re.compile(r"^(" )
r"extern +\"C\"|" exclusion_lines = re.compile(
r"(typedef +)?(struct|union|enum)( *{)?$|" r"^("
r"} *;?$|" r"extern +\"C\"|"
r"$|" r"(typedef +)?(struct|union|enum)( *{)?$|"
r"//|" r"} *;?$|"
r"#" r"$|"
r")") r"//|"
r"#"
r")"
)
self.log.debug("Looking for identifiers in {} files".format(len(header_files))) self.log.debug(
"Looking for identifiers in {} files"
.format(len(files))
)
identifiers = [] identifiers = []
for header_file in header_files: for header_file in files:
with open(header_file, "r", encoding="utf-8") as header: with open(header_file, "r", encoding="utf-8") as header:
in_block_comment = False in_block_comment = False
# The previous line varibale is used for concatenating lines
# when identifiers are formatted and spread across multiple.
previous_line = "" previous_line = ""
for line_no, line in enumerate(header): for line_no, line in enumerate(header):
@ -484,15 +500,19 @@ class NameCheck():
identifier = identifier_regex.search(line) identifier = identifier_regex.search(line)
if identifier: if not identifier:
# Find the group that matched, and append it continue
for group in identifier.groups():
if group: # Find the group that matched, and append it
identifiers.append(Match( for group in identifier.groups():
header_file, if not group:
line, continue
(line_no, identifier.start(), identifier.end()),
group)) identifiers.append(Match(
header_file,
line,
(line_no, identifier.start(), identifier.end()),
group))
return identifiers return identifiers
@ -510,8 +530,10 @@ class NameCheck():
symbols = [] symbols = []
# Back up the config and atomically compile with the full configratuion. # Back up the config and atomically compile with the full configratuion.
shutil.copy("include/mbedtls/mbedtls_config.h", shutil.copy(
"include/mbedtls/mbedtls_config.h.bak") "include/mbedtls/mbedtls_config.h",
"include/mbedtls/mbedtls_config.h.bak"
)
try: try:
# Use check=True in all subprocess calls so that failures are raised # Use check=True in all subprocess calls so that failures are raised
# as exceptions and logged. # as exceptions and logged.
@ -532,10 +554,11 @@ class NameCheck():
) )
# Perform object file analysis using nm # Perform object file analysis using nm
symbols = self.parse_symbols_from_nm( symbols = self.parse_symbols_from_nm([
["library/libmbedcrypto.a", "library/libmbedcrypto.a",
"library/libmbedtls.a", "library/libmbedtls.a",
"library/libmbedx509.a"]) "library/libmbedx509.a"
])
subprocess.run( subprocess.run(
["make", "clean"], ["make", "clean"],
@ -549,8 +572,10 @@ class NameCheck():
finally: finally:
# Put back the original config regardless of there being errors. # Put back the original config regardless of there being errors.
# Works also for keyboard interrupts. # Works also for keyboard interrupts.
shutil.move("include/mbedtls/mbedtls_config.h.bak", shutil.move(
"include/mbedtls/mbedtls_config.h") "include/mbedtls/mbedtls_config.h.bak",
"include/mbedtls/mbedtls_config.h"
)
return symbols return symbols
@ -606,9 +631,11 @@ class NameCheck():
problems += self.check_symbols_declared_in_header(quiet) problems += self.check_symbols_declared_in_header(quiet)
pattern_checks = [("macros", MACRO_PATTERN), pattern_checks = [
("enum_consts", CONSTANTS_PATTERN), ("macros", MACRO_PATTERN),
("identifiers", IDENTIFIER_PATTERN)] ("enum_consts", CONSTANTS_PATTERN),
("identifiers", IDENTIFIER_PATTERN)
]
for group, check_pattern in pattern_checks: for group, check_pattern in pattern_checks:
problems += self.check_match_pattern(quiet, group, check_pattern) problems += self.check_match_pattern(quiet, group, check_pattern)
@ -645,12 +672,11 @@ class NameCheck():
break break
if not found_symbol_declared: if not found_symbol_declared:
problems.append(SymbolNotInHeader(symbol, quiet=quiet)) problems.append(SymbolNotInHeader(symbol))
self.output_check_result("All symbols in header", problems) self.output_check_result(quiet, "All symbols in header", problems)
return len(problems) return len(problems)
def check_match_pattern(self, quiet, group_to_check, check_pattern): def check_match_pattern(self, quiet, group_to_check, check_pattern):
""" """
Perform a check that all items of a group conform to a regex pattern. Perform a check that all items of a group conform to a regex pattern.
@ -670,12 +696,10 @@ class NameCheck():
problems.append(PatternMismatch(check_pattern, item_match)) problems.append(PatternMismatch(check_pattern, item_match))
# Double underscore is a reserved identifier, never to be used # Double underscore is a reserved identifier, never to be used
if re.match(r".*__.*", item_match.name): if re.match(r".*__.*", item_match.name):
problems.append(PatternMismatch( problems.append(PatternMismatch("double underscore", item_match))
"double underscore",
item_match,
quiet=quiet))
self.output_check_result( self.output_check_result(
quiet,
"Naming patterns of {}".format(group_to_check), "Naming patterns of {}".format(group_to_check),
problems) problems)
return len(problems) return len(problems)
@ -693,7 +717,7 @@ class NameCheck():
""" """
problems = [] problems = []
# Set comprehension, equivalent to a list comprehension inside set() # Set comprehension, equivalent to a list comprehension wrapped by set()
all_caps_names = { all_caps_names = {
match.name match.name
for match for match
@ -713,20 +737,26 @@ class NameCheck():
"MBEDTLS_PSA_BUILTIN_") in all_caps_names "MBEDTLS_PSA_BUILTIN_") in all_caps_names
if not found and not typo_exclusion.search(name_match.name): if not found and not typo_exclusion.search(name_match.name):
problems.append(Typo(name_match, quiet=quiet)) problems.append(Typo(name_match))
self.output_check_result("Likely typos", problems) self.output_check_result(quiet, "Likely typos", problems)
return len(problems) return len(problems)
def output_check_result(self, name, problems): def output_check_result(self, quiet, name, problems):
""" """
Write out the PASS/FAIL status of a performed check depending on whether Write out the PASS/FAIL status of a performed check depending on whether
there were problems. there were problems.
Args:
* quiet: whether to hide detailed problem explanation.
* name: the name of the test
* problems: a List of encountered Problems
""" """
if problems: if problems:
self.set_return_code(1) self.set_return_code(1)
self.log.info("{}: FAIL\n".format(name)) self.log.info("{}: FAIL\n".format(name))
for problem in problems: for problem in problems:
problem.quiet = quiet
self.log.warning(str(problem)) self.log.warning(str(problem))
else: else:
self.log.info("{}: PASS".format(name)) self.log.info("{}: PASS".format(name))