diff --git a/tests/scripts/generate_test_code.py b/tests/scripts/generate_test_code.py index 33da990df..b2d49129e 100755 --- a/tests/scripts/generate_test_code.py +++ b/tests/scripts/generate_test_code.py @@ -23,14 +23,18 @@ Test Suite code generator. Generates a test source file using following input files: -test_suite_xyz.function - Read test functions from test suite functions file. -test_suite_xyz.data - Read test functions and their dependencies to generate - dispatch and dependency check code. -main template - Substitute generated test function dispatch code, dependency - checking code. -platform .function - Read host or target platform implementation for - dispatching test cases from .data file. -helper .function - Read common reusable functions. +test_suite_xyz.function - Read test functions from test suite + functions file. +test_suite_xyz.data - Read test functions and their + dependencies to generate dispatch and + dependency check code. +main_test.function - Template to substitute generated test + function dispatch code, dependency + checking code. +platform .function - Read host or target platform + implementation for dispatching test + cases from .data file. +helpers.function - Read common reusable functions. """ @@ -83,8 +87,8 @@ class FileWrapper(io.FileIO): super(FileWrapper, self).__init__(file_name, 'r') self.line_no = 0 - # Override the generator function in a way that works in both Python 2 - # and Python 3. + # Override the generator function in a way that works in both + # Python 2 and Python 3. def __next__(self): """ Iterator return impl. @@ -109,7 +113,8 @@ def split_dep(dep): Split NOT character '!' from dependency. Used by gen_deps() :param dep: Dependency list - :return: list of tuples where index 0 has '!' if there was a '!' before the dependency string + :return: list of tuples where index 0 has '!' if there was a '!' + before the dependency string """ return ('!', dep[1:]) if dep[0] == '!' else ('', dep) @@ -119,7 +124,8 @@ def gen_deps(deps): Generates dependency i.e. if def and endif code :param deps: List of dependencies. - :return: if defined and endif code with macro annotations for readability. + :return: if defined and endif code with macro annotations for + readability. """ dep_start = ''.join(['#if %sdefined(%s)\n' % split_dep(x) for x in deps]) dep_end = ''.join(['#endif /* %s */\n' % x for x in reversed(deps)]) @@ -129,22 +135,26 @@ def gen_deps(deps): def gen_deps_one_line(deps): """ - Generates dependency checks in one line. Useful for writing code in #else case. + Generates dependency checks in one line. Useful for writing code + in #else case. :param deps: List of dependencies. :return: ifdef code """ - defines = ('#if ' if len(deps) else '') + ' && '.join(['%sdefined(%s)' % split_dep(x) for x in deps]) + defines = '#if ' if len(deps) else '' + defines += ' && '.join(['%sdefined(%s)' % split_dep(x) for x in deps]) return defines def gen_function_wrapper(name, locals, args_dispatch): """ - Creates test function wrapper code. A wrapper has the code to unpack parameters from parameters[] array. + Creates test function wrapper code. A wrapper has the code to + unpack parameters from parameters[] array. :param name: Test function name :param locals: Local variables declaration code - :param args_dispatch: List of dispatch arguments. Ex: ['(char *)params[0]', '*((int *)params[1])'] + :param args_dispatch: List of dispatch arguments. + Ex: ['(char *)params[0]', '*((int *)params[1])'] :return: Test function wrapper. """ # Then create the wrapper @@ -200,7 +210,8 @@ def parse_until_pattern(funcs_f, end_regex): break headers += line else: - raise InvalidFileFormat("file: %s - end pattern [%s] not found!" % (funcs_f.name, end_regex)) + raise InvalidFileFormat("file: %s - end pattern [%s] not found!" % + (funcs_f.name, end_regex)) return headers @@ -220,7 +231,8 @@ def parse_suite_deps(funcs_f): if re.search(END_DEP_REGEX, line): break else: - raise InvalidFileFormat("file: %s - end dependency pattern [%s] not found!" % (funcs_f.name, END_DEP_REGEX)) + raise InvalidFileFormat("file: %s - end dependency pattern [%s]" + " not found!" % (funcs_f.name, END_DEP_REGEX)) return deps @@ -246,8 +258,10 @@ def parse_function_signature(line): """ Parsing function signature - :param line: Line from .functions file that has a function signature. - :return: function name, argument list, local variables for wrapper function and argument dispatch code. + :param line: Line from .functions file that has a function + signature. + :return: function name, argument list, local variables for + wrapper function and argument dispatch code. """ args = [] locals = '' @@ -271,13 +285,16 @@ def parse_function_signature(line): elif re.search('HexParam_t\s*\*\s*.*', arg.strip()): args.append('hex') # create a structure + pointer_initializer = '(uint8_t *) params[%d]' % arg_idx + len_initializer = '*( (uint32_t *) params[%d] )' % (arg_idx+1) locals += """ HexParam_t hex%d = {%s, %s}; -""" % (arg_idx, '(uint8_t *) params[%d]' % arg_idx, '*( (uint32_t *) params[%d] )' % (arg_idx + 1)) +""" % (arg_idx, pointer_initializer, len_initializer) args_dispatch.append('&hex%d' % arg_idx) arg_idx += 1 else: - raise ValueError("Test function arguments can only be 'int', 'char *' or 'HexParam_t'\n%s" % line) + raise ValueError("Test function arguments can only be 'int', " + "'char *' or 'HexParam_t'\n%s" % line) arg_idx += 1 return name, args, locals, args_dispatch @@ -285,7 +302,8 @@ def parse_function_signature(line): def parse_function_code(funcs_f, deps, suite_deps): """ - Parses out a function from function file object and generates function and dispatch code. + Parses out a function from function file object and generates + function and dispatch code. :param funcs_f: file object of the functions file. :param deps: List of dependencies @@ -308,14 +326,16 @@ def parse_function_code(funcs_f, deps, suite_deps): name = 'test_' + name break else: - raise InvalidFileFormat("file: %s - Test functions not found!" % funcs_f.name) + raise InvalidFileFormat("file: %s - Test functions not found!" % + funcs_f.name) for line in funcs_f: if re.search(END_CASE_REGEX, line): break code += line else: - raise InvalidFileFormat("file: %s - end case pattern [%s] not found!" % (funcs_f.name, END_CASE_REGEX)) + raise InvalidFileFormat("file: %s - end case pattern [%s] not " + "found!" % (funcs_f.name, END_CASE_REGEX)) # Add exit label if not present if code.find('exit:') == -1: @@ -336,8 +356,9 @@ def parse_functions(funcs_f): Returns functions code pieces :param funcs_f: file object of the functions file. - :return: List of test suite dependencies, test function dispatch code, function code and - a dict with function identifiers and arguments info. + :return: List of test suite dependencies, test function dispatch + code, function code and a dict with function identifiers + and arguments info. """ suite_headers = '' suite_helpers = '' @@ -358,7 +379,8 @@ def parse_functions(funcs_f): suite_deps += deps elif re.search(BEGIN_CASE_REGEX, line): deps = parse_function_deps(line) - func_name, args, func_code, func_dispatch = parse_function_code(funcs_f, deps, suite_deps) + func_name, args, func_code, func_dispatch =\ + parse_function_code(funcs_f, deps, suite_deps) suite_functions += func_code # Generate dispatch code and enumeration info if func_name in func_info: @@ -378,8 +400,9 @@ def parse_functions(funcs_f): def escaped_split(str, ch): """ Split str on character ch but ignore escaped \{ch} - Since return value is used to write back to the intermediate data file. - Any escape characters in the input are retained in the output. + Since, return value is used to write back to the intermediate + data file, any escape characters in the input are retained in the + output. :param str: String to split :param ch: split character @@ -407,7 +430,8 @@ def parse_test_data(data_f, debug=False): Parses .data file :param data_f: file object of the data file. - :return: Generator that yields test name, function name, dependency list and function argument list. + :return: Generator that yields test name, function name, + dependency list and function argument list. """ STATE_READ_NAME = 0 STATE_READ_ARGS = 1 @@ -422,9 +446,10 @@ def parse_test_data(data_f, debug=False): # Blank line indicates end of test if len(line) == 0: if state == STATE_READ_ARGS: - raise GeneratorInputError("[%s:%d] Newline before arguments. " \ - "Test function and arguments missing for %s" % \ - (data_f.name, data_f.line_no, name)) + raise GeneratorInputError("[%s:%d] Newline before arguments. " + "Test function and arguments " + "missing for %s" % + (data_f.name, data_f.line_no, name)) continue if state == STATE_READ_NAME: @@ -435,7 +460,8 @@ def parse_test_data(data_f, debug=False): # Check dependencies m = re.search('depends_on\:(.*)', line) if m: - deps = [x.strip() for x in m.group(1).split(':') if len(x.strip())] + deps = [x.strip() for x in m.group(1).split(':') if len( + x.strip())] else: # Read test vectors parts = escaped_split(line, ':') @@ -445,9 +471,9 @@ def parse_test_data(data_f, debug=False): deps = [] state = STATE_READ_NAME if state == STATE_READ_ARGS: - raise GeneratorInputError("[%s:%d] Newline before arguments. " \ - "Test function and arguments missing for %s" % \ - (data_f.name, data_f.line_no, name)) + raise GeneratorInputError("[%s:%d] Newline before arguments. " + "Test function and arguments missing for " + "%s" % (data_f.name, data_f.line_no, name)) def gen_dep_check(dep_id, dep): @@ -459,7 +485,8 @@ def gen_dep_check(dep_id, dep): :return: Dependency check code """ if dep_id < 0: - raise GeneratorInputError("Dependency Id should be a positive integer.") + raise GeneratorInputError("Dependency Id should be a positive " + "integer.") noT, dep = ('!', dep[1:]) if dep[0] == '!' else ('', dep) if len(dep) == 0: raise GeneratorInputError("Dependency should not be an empty string.") @@ -485,7 +512,8 @@ def gen_expression_check(exp_id, exp): :return: Expression check code """ if exp_id < 0: - raise GeneratorInputError("Expression Id should be a positive integer.") + raise GeneratorInputError("Expression Id should be a positive " + "integer.") if len(exp) == 0: raise GeneratorInputError("Expression should not be an empty string.") exp_code = ''' @@ -504,7 +532,8 @@ def write_deps(out_data_f, test_deps, unique_deps): :param out_data_f: Output intermediate data file :param test_deps: Dependencies - :param unique_deps: Mutable list to track unique dependencies that are global to this re-entrant function. + :param unique_deps: Mutable list to track unique dependencies + that are global to this re-entrant function. :return: returns dependency check code. """ dep_check_code = '' @@ -530,7 +559,8 @@ def write_parameters(out_data_f, test_args, func_args, unique_expressions): :param out_data_f: Output intermediate data file :param test_args: Test parameters :param func_args: Function arguments - :param unique_expressions: Mutable list to track unique expressions that are global to this re-entrant function. + :param unique_expressions: Mutable list to track unique + expressions that are global to this re-entrant function. :return: Returns expression check code. """ expression_code = '' @@ -538,13 +568,14 @@ def write_parameters(out_data_f, test_args, func_args, unique_expressions): typ = func_args[i] val = test_args[i] - # check if val is a non literal int val - if typ == 'int' and not re.match('(\d+$)|((0x)?[0-9a-fA-F]+$)', val): # its an expression + # check if val is a non literal int val (i.e. an expression) + if typ == 'int' and not re.match('(\d+$)|((0x)?[0-9a-fA-F]+$)', val): typ = 'exp' if val not in unique_expressions: unique_expressions.append(val) - # exp_id can be derived from len(). But for readability and consistency with case of existing let's - # use index(). + # exp_id can be derived from len(). But for + # readability and consistency with case of existing + # let's use index(). exp_id = unique_expressions.index(val) expression_code += gen_expression_check(exp_id, val) val = exp_id @@ -559,10 +590,12 @@ def gen_suite_deps_checks(suite_deps, dep_check_code, expression_code): """ Adds preprocessor checks for test suite dependencies. - :param suite_deps: Test suite dependencies read from the .functions file. + :param suite_deps: Test suite dependencies read from the + .functions file. :param dep_check_code: Dependency check code :param expression_code: Expression check code - :return: Dependency and expression code guarded by test suite dependencies. + :return: Dependency and expression code guarded by test suite + dependencies. """ if len(suite_deps): ifdef = gen_deps_one_line(suite_deps) @@ -581,11 +614,13 @@ def gen_suite_deps_checks(suite_deps, dep_check_code, expression_code): def gen_from_test_data(data_f, out_data_f, func_info, suite_deps): """ - Generates dependency checks, expression code and intermediate data file from test data file. + Generates dependency checks, expression code and intermediate + data file from test data file. :param data_f: Data file object :param out_data_f:Output intermediate data file - :param func_info: Dict keyed by function and with function id and arguments info + :param func_info: Dict keyed by function and with function id + and arguments info :param suite_deps: Test suite deps :return: Returns dependency and expression check code """ @@ -593,7 +628,8 @@ def gen_from_test_data(data_f, out_data_f, func_info, suite_deps): unique_expressions = [] dep_check_code = '' expression_code = '' - for test_name, function_name, test_deps, test_args in parse_test_data(data_f): + for test_name, function_name, test_deps, test_args in parse_test_data( + data_f): out_data_f.write(test_name + '\n') # Write deps @@ -602,24 +638,29 @@ def gen_from_test_data(data_f, out_data_f, func_info, suite_deps): # Write test function name test_function_name = 'test_' + function_name if test_function_name not in func_info: - raise GeneratorInputError("Function %s not found!" % test_function_name) + raise GeneratorInputError("Function %s not found!" % + test_function_name) func_id, func_args = func_info[test_function_name] out_data_f.write(str(func_id)) # Write parameters if len(test_args) != len(func_args): - raise GeneratorInputError("Invalid number of arguments in test %s. See function %s signature." % (test_name, - function_name)) - expression_code += write_parameters(out_data_f, test_args, func_args, unique_expressions) + raise GeneratorInputError("Invalid number of arguments in test " + "%s. See function %s signature." % ( + test_name, function_name)) + expression_code += write_parameters(out_data_f, test_args, func_args, + unique_expressions) # Write a newline as test case separator out_data_f.write('\n') - dep_check_code, expression_code = gen_suite_deps_checks(suite_deps, dep_check_code, expression_code) + dep_check_code, expression_code = gen_suite_deps_checks( + suite_deps, dep_check_code, expression_code) return dep_check_code, expression_code -def generate_code(funcs_file, data_file, template_file, platform_file, help_file, suites_dir, c_file, out_data_file): +def generate_code(funcs_file, data_file, template_file, platform_file, + help_file, suites_dir, c_file, out_data_file): """ Generate mbed-os test code. @@ -645,19 +686,23 @@ def generate_code(funcs_file, data_file, template_file, platform_file, help_file snippets = {'generator_script' : os.path.basename(__file__)} # Read helpers - with open(help_file, 'r') as help_f, open(platform_file, 'r') as platform_f: + with open(help_file, 'r') as help_f, open(platform_file, 'r') as \ + platform_f: snippets['test_common_helper_file'] = help_file snippets['test_common_helpers'] = help_f.read() snippets['test_platform_file'] = platform_file - snippets['platform_code'] = platform_f.read().replace('DATA_FILE', - out_data_file.replace('\\', '\\\\')) # escape '\' + snippets['platform_code'] = platform_f.read().replace( + 'DATA_FILE', out_data_file.replace('\\', '\\\\')) # escape '\' # Function code - with FileWrapper(funcs_file) as funcs_f, FileWrapper(data_file) as data_f, open(out_data_file, 'w') as out_data_f: - suite_deps, dispatch_code, func_code, func_info = parse_functions(funcs_f) + with FileWrapper(funcs_file) as funcs_f, FileWrapper(data_file) as \ + data_f, open(out_data_file, 'w') as out_data_f: + suite_deps, dispatch_code, func_code, func_info = parse_functions( + funcs_f) snippets['functions_code'] = func_code snippets['dispatch_code'] = dispatch_code - dep_check_code, expression_code = gen_from_test_data(data_f, out_data_f, func_info, suite_deps) + dep_check_code, expression_code = gen_from_test_data( + data_f, out_data_f, func_info, suite_deps) snippets['dep_check_code'] = dep_check_code snippets['expression_code'] = expression_code @@ -671,7 +716,8 @@ def generate_code(funcs_file, data_file, template_file, platform_file, help_file with open(template_file, 'r') as template_f, open(c_file, 'w') as c_f: line_no = 1 for line in template_f.readlines(): - snippets['line_no'] = line_no + 1 # Increment as it sets next line number + # Update line number. +1 as #line directive sets next line number + snippets['line_no'] = line_no + 1 code = line.format(**snippets) c_f.write(code) line_no += 1 @@ -683,7 +729,8 @@ def check_cmd(): :return: """ - parser = argparse.ArgumentParser(description='Generate code for mbed-os tests.') + parser = argparse.ArgumentParser( + description='Generate code for mbed-os tests.') parser.add_argument("-f", "--functions-file", dest="funcs_file", @@ -741,8 +788,9 @@ def check_cmd(): if not os.path.exists(d): os.makedirs(d) - generate_code(args.funcs_file, args.data_file, args.template_file, args.platform_file, - args.help_file, args.suites_dir, out_c_file, out_data_file) + generate_code(args.funcs_file, args.data_file, args.template_file, + args.platform_file, args.help_file, args.suites_dir, + out_c_file, out_data_file) if __name__ == "__main__":