diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 746521703..6101a1c28 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -61,24 +61,32 @@ def analyze_coverage(results, outcomes): # fixed this branch to have full coverage of test cases. results.warning('Test case not executed: {}', key) -def analyze_driver_vs_reference(outcomes, component_ref, component_driver, ignored_tests): +def analyze_driver_vs_reference(outcomes, component_ref, component_driver, + ignored_suites, ignored_test=None): """Check that all tests executed in the reference component are also executed in the corresponding driver component. - Skip test suites provided in ignored_tests list. + Skip: + - full test suites provided in ignored_suites list + - only some specific test inside a test suite, for which the corresponding + output string is provided """ available = check_test_cases.collect_available_test_cases() result = True for key in available: # Skip ignored test suites - test_suite = key.split(';')[0] # retrieve test suit name - test_suite = test_suite.split('.')[0] # retrieve main part of test suit name - if test_suite in ignored_tests: + full_test_suite = key.split(';')[0] # retrieve full test suit name + test_string = key.split(';')[1] # retrieve the text string of this test + test_suite = full_test_suite.split('.')[0] # retrieve main part of test suit name + if test_suite in ignored_suites: continue # Continue if test was not executed by any component hits = outcomes[key].hits() if key in outcomes else 0 if hits == 0: continue + if ((full_test_suite in ignored_test) and + (test_string in ignored_test[full_test_suite])): + continue # Search for tests that run in reference component and not in driver component driver_test_passed = False reference_test_passed = False @@ -129,13 +137,14 @@ def do_analyze_coverage(outcome_file, args): def do_analyze_driver_vs_reference(outcome_file, args): """Perform driver vs reference analyze.""" - ignored_tests = ['test_suite_' + x for x in args['ignored_suites']] + ignored_suites = ['test_suite_' + x for x in args['ignored_suites']] outcomes = read_outcome_file(outcome_file) print("\n*** Analyze driver {} vs reference {} ***\n".format( args['component_driver'], args['component_ref'])) return analyze_driver_vs_reference(outcomes, args['component_ref'], - args['component_driver'], ignored_tests) + args['component_driver'], ignored_suites, + args['ignored_tests']) # List of tasks with a function that can handle this task and additional arguments if required TASKS = { @@ -154,7 +163,11 @@ TASKS = { 'ignored_suites': [ 'shax', 'mdx', # the software implementations that are being excluded 'md', # the legacy abstraction layer that's being excluded - ]}}, + ], + 'ignored_tests': { + } + } + }, 'analyze_driver_vs_reference_ecdsa': { 'test_function': do_analyze_driver_vs_reference, 'args': { @@ -170,7 +183,11 @@ TASKS = { 'x509write', # #6858 'debug', # #6860 'ssl', # #6860 - ]}}, + ], + 'ignored_tests': { + } + } + }, } def main():