diff --git a/include/SDL3/SDL_test_harness.h b/include/SDL3/SDL_test_harness.h index 263071fbd..70adc3585 100644 --- a/include/SDL3/SDL_test_harness.h +++ b/include/SDL3/SDL_test_harness.h @@ -117,10 +117,11 @@ char *SDLTest_GenerateRunSeed(const int length); * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one. * \param filter Filter specification. NULL disables. Case sensitive. * \param testIterations Number of iterations to run each test case. + * \param randomOrder allow to run suites and tests in random order when there is no filter * * \returns the test run result: 0 when all tests passed, 1 if any tests failed. */ -int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations); +int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations, SDL_bool randomOrder); /* Ends C function definitions when using C++ */ diff --git a/src/test/SDL_test_harness.c b/src/test/SDL_test_harness.c index e306e4630..f91d4903e 100644 --- a/src/test/SDL_test_harness.c +++ b/src/test/SDL_test_harness.c @@ -365,10 +365,11 @@ static float GetClock(void) * \param userExecKey Custom execution key provided by user, or 0 to autogenerate one. * \param filter Filter specification. NULL disables. Case sensitive. * \param testIterations Number of iterations to run each test case. + * \param randomOrder allow to run suites and tests in random order when there is no filter * * \returns Test run result; 0 when all tests passed, 1 if any tests failed. */ -int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations) +int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations, SDL_bool randomOrder) { int totalNumberOfTests = 0; int failedNumberOfTests = 0; @@ -404,6 +405,9 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user int countSum = 0; const SDLTest_TestCaseReference **failedTests; char generatedSeed[16 + 1]; + int nbSuites = 0; + int i = 0; + int *arraySuites = NULL; /* Sanitize test iterations */ if (testIterations < 1) { @@ -509,11 +513,59 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user SDL_free((void *)failedTests); return 2; } + + randomOrder = SDL_FALSE; + } + + /* Number of test suites */ + while (testSuites[nbSuites]) { + nbSuites++; + } + + arraySuites = SDL_malloc(nbSuites * sizeof(int)); + if (!arraySuites) { + return SDL_OutOfMemory(); + } + for (i = 0; i < nbSuites; i++) { + arraySuites[i] = i; + } + + /* Mix the list of suites to run them in random order */ + { + /* Exclude last test "subsystemsTestSuite" which is said to interfer with other tests */ + nbSuites--; + + if (userExecKey != 0) { + execKey = userExecKey; + } else { + /* dummy values to have random numbers working */ + execKey = SDLTest_GenerateExecKey(runSeed, "random testSuites", "initialisation", 1); + } + + /* Initialize fuzzer */ + SDLTest_FuzzerInit(execKey); + + i = 100; + while (i--) { + int a, b; + int tmp; + a = SDLTest_RandomIntegerInRange(0, nbSuites - 1); + b = SDLTest_RandomIntegerInRange(0, nbSuites - 1); + /* Swap */ + if (randomOrder) { + tmp = arraySuites[b]; + arraySuites[b] = arraySuites[a]; + arraySuites[a] = tmp; + } + } + + /* re-add last lest */ + nbSuites++; } /* Loop over all suites */ - suiteCounter = 0; - while (testSuites[suiteCounter]) { + for (i = 0; i < nbSuites; i++) { + suiteCounter = arraySuites[i];; testSuite = testSuites[suiteCounter]; currentSuiteName = (testSuite->name ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT); suiteCounter++; @@ -527,6 +579,36 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user currentSuiteName); } else { + int nbTestCases = 0; + int *arrayTestCases; + int j; + while (testSuite->testCases[nbTestCases]) { + nbTestCases++; + } + + arrayTestCases = SDL_malloc(nbTestCases * sizeof(int)); + if (!arrayTestCases) { + return SDL_OutOfMemory(); + } + for (j = 0; j < nbTestCases; j++) { + arrayTestCases[j] = j; + } + + /* Mix the list of testCases to run them in random order */ + j = 100; + while (j--) { + int a, b; + int tmp; + a = SDLTest_RandomIntegerInRange(0, nbTestCases - 1); + b = SDLTest_RandomIntegerInRange(0, nbTestCases - 1); + /* Swap */ + if (randomOrder) { + tmp = arrayTestCases[b]; + arrayTestCases[b] = arrayTestCases[a]; + arrayTestCases[a] = tmp; + } + } + /* Reset per-suite counters */ testFailedCount = 0; testPassedCount = 0; @@ -541,8 +623,8 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user currentSuiteName); /* Loop over all test cases */ - testCounter = 0; - while (testSuite->testCases[testCounter]) { + for (j = 0; j < nbTestCases; j++) { + testCounter = arrayTestCases[j]; testCase = testSuite->testCases[testCounter]; currentTestName = (testCase->name ? testCase->name : SDLTEST_INVALID_NAME_FORMAT); testCounter++; @@ -657,9 +739,13 @@ int SDLTest_RunSuites(SDLTest_TestSuiteReference *testSuites[], const char *user SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount); SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, COLOR_RED "Failed" COLOR_END); } + + SDL_free(arrayTestCases); } } + SDL_free(arraySuites); + /* Take time - run end */ runEndSeconds = GetClock(); runtime = runEndSeconds - runStartSeconds; diff --git a/test/testautomation.c b/test/testautomation.c index a8de28bf9..42dfeef58 100644 --- a/test/testautomation.c +++ b/test/testautomation.c @@ -75,6 +75,7 @@ int main(int argc, char *argv[]) int i, done; SDL_Event event; int list = 0; + SDL_bool randomOrder = SDL_FALSE; /* Initialize test framework */ state = SDLTest_CommonCreateState(argv, SDL_INIT_VIDEO | SDL_INIT_AUDIO); @@ -118,10 +119,21 @@ int main(int argc, char *argv[]) } else if (SDL_strcasecmp(argv[i], "--list") == 0) { consumed = 1; list = 1; + } else if (SDL_strcasecmp(argv[i], "--random-order") == 0) { + consumed = 1; + randomOrder = SDL_TRUE; } + } if (consumed < 0) { - static const char *options[] = { "[--iterations #]", "[--execKey #]", "[--seed string]", "[--filter suite_name|test_name]", "[--list]", NULL }; + static const char *options[] = { + "[--iterations #]", + "[--execKey #]", + "[--seed string]", + "[--filter suite_name|test_name]", + "[--list]", + "[--random-order]", + NULL }; SDLTest_CommonLogUsage(state, argv[0], options); quit(1); } @@ -157,7 +169,7 @@ int main(int argc, char *argv[]) } /* Call Harness */ - result = SDLTest_RunSuites(testSuites, userRunSeed, userExecKey, filter, testIterations); + result = SDLTest_RunSuites(testSuites, userRunSeed, userExecKey, filter, testIterations, randomOrder); /* Empty event queue */ done = 0;