From 4e95590ae79ba9c026eec992f3e307f4cf16f8a4 Mon Sep 17 00:00:00 2001
From: Przemek Stekiel <przemyslaw.stekiel@mobica.com>
Date: Fri, 21 Oct 2022 13:42:08 +0200
Subject: [PATCH] analyze_outcomes.py: Add test coverage regresion analyze for
 driver only builds

Signed-off-by: Przemek Stekiel <przemyslaw.stekiel@mobica.com>
---
 tests/scripts/analyze_outcomes.py | 77 ++++++++++++++++++++++++++++---
 1 file changed, 71 insertions(+), 6 deletions(-)

diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py
index d06a0596f..f5d2ac131 100755
--- a/tests/scripts/analyze_outcomes.py
+++ b/tests/scripts/analyze_outcomes.py
@@ -60,6 +60,41 @@ def analyze_coverage(results, outcomes):
             # fixed this branch to have full coverage of test cases.
             results.warning('Test case not executed: {}', key)
 
+def analyze_driver_vs_reference(outcomes, components, ignored_tests):
+    """Check that all tests executed in the reference component are also
+    executed in the corresponding driver component.
+    Skip test suits provided in ignored_tests list.
+    """
+    driver_component = components[0]
+    reference_component = components[1]
+    available = check_test_cases.collect_available_test_cases()
+    result = True
+
+    for key in available:
+        # Skip ignored test suites
+        test_suit = key.split(';')[0] # retrieve test suit name
+        test_suit = test_suit.split('.')[0] # retrieve main part of test suit name
+        if(test_suit in ignored_tests):
+            continue
+        # Continue if test was not executed by any component
+        hits = outcomes[key].hits() if key in outcomes else 0
+        if(hits == 0):
+            continue
+        # Search for tests that run in reference component and not in driver component
+        driver_test_passed = False
+        reference_test_passed = False
+        for entry in outcomes[key].successes:
+            if(driver_component in entry):
+                driver_test_passed = True
+            if(reference_component in entry):
+                reference_test_passed = True
+        #if(driver_test_passed == True and reference_test_passed == False):
+        #    print('{}: driver: passed; reference: skipped'.format(key))
+        if(driver_test_passed == False and reference_test_passed == True):
+            print('{}: driver: skipped/failed; reference: passed'.format(key))
+            result = False
+    return result
+
 def analyze_outcomes(outcomes):
     """Run all analyses on the given outcome collection."""
     results = Results()
@@ -87,20 +122,50 @@ by a semicolon.
                 outcomes[key].failures.append(setup)
     return outcomes
 
-def analyze_outcome_file(outcome_file):
-    """Analyze the given outcome file."""
+def do_analyze_coverage(outcome_file):
+    """Perform coverage analyze."""
     outcomes = read_outcome_file(outcome_file)
-    return analyze_outcomes(outcomes)
+    results = analyze_outcomes(outcomes)
+    return (True if results.error_count == 0 else False)
+
+def do_analyze_driver_vs_reference(outcome_file, components, ignored_tests):
+    """Perform driver vs reference analyze."""
+    # We need exactly 2 components to analyze (first driver and second reference)
+    if(len(components) != 2 or "accel" not in components[0] or "reference" not in components[1]):
+        print('Error: Wrong component list. Exactly 2 components are required (driver,reference). ')
+        return False
+    outcomes = read_outcome_file(outcome_file)
+    return analyze_driver_vs_reference(outcomes, components, ignored_tests)
 
 def main():
     try:
         parser = argparse.ArgumentParser(description=__doc__)
-        parser.add_argument('outcomes', metavar='OUTCOMES.CSV',
+        parser.add_argument('--outcomes', metavar='OUTCOMES.CSV',
                             help='Outcome file to analyze')
+        parser.add_argument('--task',
+                            help='Analyze to be done: analyze_coverage or analyze_driver_vs_reference')
+        parser.add_argument('--components',
+                            help='List of test components to compare. Must be exactly 2 in valid order: driver,reference. '
+                            'Apply only for analyze_driver_vs_reference task.')
+        parser.add_argument('--ignore',
+                            help='List of test suits to ignore. Apply only for analyze_driver_vs_reference task.')
         options = parser.parse_args()
-        results = analyze_outcome_file(options.outcomes)
-        if results.error_count > 0:
+
+        result = False
+
+        if(options.task == 'analyze_coverage'):
+            result = do_analyze_coverage(options.outcomes)
+        elif(options.task == 'analyze_driver_vs_reference'):
+            components_list = options.components.split(',')
+            ignored_tests_list = options.ignore.split(',')
+            ignored_tests_list = ['test_suite_' + x for x in ignored_tests_list]
+            result = do_analyze_driver_vs_reference(options.outcomes, components_list, ignored_tests_list)
+        else:
+            print('Error: Unknown task: {}'.format(options.task))
+
+        if(result == False):
             sys.exit(1)
+        print("SUCCESS :-)")
     except Exception: # pylint: disable=broad-except
         # Print the backtrace and exit explicitly with our chosen status.
         traceback.print_exc()