From 07bdcc2b0dd181823702cc8f842a0b47c4cf3324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Fri, 11 Aug 2023 14:59:03 +0100 Subject: [PATCH] Add allow list for non-executed test cases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The allow list explicits which test cases are allowed to not be executed when testing. This may be, for example, because a feature is yet to be developed but the test for that feature is already in our code base. Signed-off-by: Tomás González --- tests/scripts/analyze_outcomes.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index c6891bb43..fde07159e 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -73,15 +73,18 @@ def execute_reference_driver_tests(ref_component, driver_component, outcome_file Results.log("Error: failed to run reference/driver components") sys.exit(ret_val) -def analyze_coverage(results, outcomes): +def analyze_coverage(results, outcomes, allow_list): """Check that all available test cases are executed at least once.""" available = check_test_cases.collect_available_test_cases() for key in available: hits = outcomes[key].hits() if key in outcomes else 0 - if hits == 0: + if hits == 0 and key not in allow_list: # Make this a warning, not an error, as long as we haven't # fixed this branch to have full coverage of test cases. results.warning('Test case not executed: {}', key) + elif hits != 0 and key in allow_list: + # Test Case should be removed from the allow list. + results.warning('Allow listed test case was executed: {}', key) def analyze_driver_vs_reference(outcomes, component_ref, component_driver, ignored_suites, ignored_test=None): @@ -122,10 +125,10 @@ def analyze_driver_vs_reference(outcomes, component_ref, component_driver, result = False return result -def analyze_outcomes(outcomes): +def analyze_outcomes(outcomes, allow_list): """Run all analyses on the given outcome collection.""" results = Results() - analyze_coverage(results, outcomes) + analyze_coverage(results, outcomes, allow_list) return results def read_outcome_file(outcome_file): @@ -151,10 +154,9 @@ by a semicolon. def do_analyze_coverage(outcome_file, args): """Perform coverage analysis.""" - del args # unused outcomes = read_outcome_file(outcome_file) Results.log("\n*** Analyze coverage ***\n") - results = analyze_outcomes(outcomes) + results = analyze_outcomes(outcomes, args['allow_list']) return results.error_count == 0 def do_analyze_driver_vs_reference(outcome_file, args): @@ -175,7 +177,9 @@ def do_analyze_driver_vs_reference(outcome_file, args): TASKS = { 'analyze_coverage': { 'test_function': do_analyze_coverage, - 'args': {} + 'args': { + 'allow_list': [], + } }, # There are 2 options to use analyze_driver_vs_reference_xxx locally: # 1. Run tests and then analysis: