diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index c6891bb43..fde07159e 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -73,15 +73,18 @@ def execute_reference_driver_tests(ref_component, driver_component, outcome_file Results.log("Error: failed to run reference/driver components") sys.exit(ret_val) -def analyze_coverage(results, outcomes): +def analyze_coverage(results, outcomes, allow_list): """Check that all available test cases are executed at least once.""" available = check_test_cases.collect_available_test_cases() for key in available: hits = outcomes[key].hits() if key in outcomes else 0 - if hits == 0: + if hits == 0 and key not in allow_list: # Make this a warning, not an error, as long as we haven't # fixed this branch to have full coverage of test cases. results.warning('Test case not executed: {}', key) + elif hits != 0 and key in allow_list: + # Test Case should be removed from the allow list. + results.warning('Allow listed test case was executed: {}', key) def analyze_driver_vs_reference(outcomes, component_ref, component_driver, ignored_suites, ignored_test=None): @@ -122,10 +125,10 @@ def analyze_driver_vs_reference(outcomes, component_ref, component_driver, result = False return result -def analyze_outcomes(outcomes): +def analyze_outcomes(outcomes, allow_list): """Run all analyses on the given outcome collection.""" results = Results() - analyze_coverage(results, outcomes) + analyze_coverage(results, outcomes, allow_list) return results def read_outcome_file(outcome_file): @@ -151,10 +154,9 @@ by a semicolon. def do_analyze_coverage(outcome_file, args): """Perform coverage analysis.""" - del args # unused outcomes = read_outcome_file(outcome_file) Results.log("\n*** Analyze coverage ***\n") - results = analyze_outcomes(outcomes) + results = analyze_outcomes(outcomes, args['allow_list']) return results.error_count == 0 def do_analyze_driver_vs_reference(outcome_file, args): @@ -175,7 +177,9 @@ def do_analyze_driver_vs_reference(outcome_file, args): TASKS = { 'analyze_coverage': { 'test_function': do_analyze_coverage, - 'args': {} + 'args': { + 'allow_list': [], + } }, # There are 2 options to use analyze_driver_vs_reference_xxx locally: # 1. Run tests and then analysis: