diff --git a/include/catch_runner.hpp b/include/catch_runner.hpp index e6303d88..c9f43ed2 100644 --- a/include/catch_runner.hpp +++ b/include/catch_runner.hpp @@ -170,6 +170,9 @@ namespace Catch { try { config(); // Force config to be constructed + + std::srand( m_configData.rngSeed ); + Runner runner( m_config ); // Handle list request diff --git a/include/external/clara.h b/include/external/clara.h index d8e83ac5..dba26a95 100644 --- a/include/external/clara.h +++ b/include/external/clara.h @@ -791,7 +791,7 @@ namespace Clara { if( it == itEnd ) { if( token.type == Parser::Token::Positional || !m_throwOnUnrecognisedTokens ) unusedTokens.push_back( token ); - else if( m_throwOnUnrecognisedTokens ) + else if( errors.empty() && m_throwOnUnrecognisedTokens ) errors.push_back( "unrecognised option: " + token.data ); } } diff --git a/include/internal/catch_commandline.hpp b/include/internal/catch_commandline.hpp index c7c33845..a17059f4 100644 --- a/include/internal/catch_commandline.hpp +++ b/include/internal/catch_commandline.hpp @@ -29,7 +29,28 @@ namespace Catch { config.warnings = static_cast( config.warnings | WarnAbout::NoAssertions ); else throw std::runtime_error( "Unrecognised warning: '" + _warning + "'" ); - + } + inline void setOrder( ConfigData& config, std::string const& order ) { + if( startsWith( "declared", order ) ) + config.runOrder = RunTests::InDeclarationOrder; + else if( startsWith( "lexical", order ) ) + config.runOrder = RunTests::InLexicographicalOrder; + else if( startsWith( "random", order ) ) + config.runOrder = RunTests::InRandomOrder; + else + throw std::runtime_error( "Unrecognised ordering: '" + order + "'" ); + } + inline void setRngSeed( ConfigData& config, std::string const& seed ) { + if( seed == "time" ) { + config.rngSeed = static_cast( std::time(0) ); + } + else { + std::stringstream ss; + ss << seed; + ss >> config.rngSeed; + if( ss.fail() ) + throw std::runtime_error( "Argment to --rng-seed should be the word 'time' or a number" ); + } } inline void setVerbosity( ConfigData& config, int level ) { // !TBD: accept strings? @@ -137,10 +158,18 @@ namespace Catch { .describe( "list all/matching test cases names only" ) .bind( &ConfigData::listTestNamesOnly ); - cli["--list-reporters"] + cli["--list-reporters"] .describe( "list all reporters" ) .bind( &ConfigData::listReporters ); + cli["--order"] + .describe( "test case order (defaults to decl)" ) + .bind( &setOrder, "decl|lex|rand" ); + + cli["--rng-seed"] + .describe( "set a specific seed for random numbers" ) + .bind( &setRngSeed, "'time'|number" ); + return cli; } diff --git a/include/internal/catch_config.hpp b/include/internal/catch_config.hpp index 1217b9a9..37ca6579 100644 --- a/include/internal/catch_config.hpp +++ b/include/internal/catch_config.hpp @@ -17,6 +17,7 @@ #include #include #include +#include #ifndef CATCH_CONFIG_CONSOLE_WIDTH #define CATCH_CONFIG_CONSOLE_WIDTH 80 @@ -37,9 +38,11 @@ namespace Catch { showHelp( false ), showInvisibles( false ), abortAfter( -1 ), + rngSeed( 0 ), verbosity( Verbosity::Normal ), warnings( WarnAbout::Nothing ), - showDurations( ShowDurations::DefaultForReporter ) + showDurations( ShowDurations::DefaultForReporter ), + runOrder( RunTests::InDeclarationOrder ) {} bool listTests; @@ -54,10 +57,12 @@ namespace Catch { bool showInvisibles; int abortAfter; + unsigned int rngSeed; Verbosity::Level verbosity; WarnAbout::What warnings; ShowDurations::OrNot showDurations; + RunTests::InWhatOrder runOrder; std::string reporterName; std::string outputFilename; @@ -141,7 +146,8 @@ namespace Catch { virtual bool includeSuccessfulResults() const { return m_data.showSuccessfulTests; } virtual bool warnAboutMissingAssertions() const { return m_data.warnings & WarnAbout::NoAssertions; } virtual ShowDurations::OrNot showDurations() const { return m_data.showDurations; } - + virtual RunTests::InWhatOrder runOrder() const { return m_data.runOrder; } + virtual unsigned int rngSeed() const { return m_data.rngSeed; } private: ConfigData m_data; diff --git a/include/internal/catch_interfaces_config.h b/include/internal/catch_interfaces_config.h index c7610d99..5f30ac1d 100644 --- a/include/internal/catch_interfaces_config.h +++ b/include/internal/catch_interfaces_config.h @@ -32,6 +32,11 @@ namespace Catch { Always, Never }; }; + struct RunTests { enum InWhatOrder { + InDeclarationOrder, + InLexicographicalOrder, + InRandomOrder + }; }; class TestSpec; @@ -49,6 +54,8 @@ namespace Catch { virtual bool showInvisibles() const = 0; virtual ShowDurations::OrNot showDurations() const = 0; virtual TestSpec const& testSpec() const = 0; + virtual RunTests::InWhatOrder runOrder() const = 0; + virtual unsigned int rngSeed() const = 0; }; } diff --git a/include/internal/catch_test_case_registry_impl.hpp b/include/internal/catch_test_case_registry_impl.hpp index ecffa2e5..ad733dbc 100644 --- a/include/internal/catch_test_case_registry_impl.hpp +++ b/include/internal/catch_test_case_registry_impl.hpp @@ -59,7 +59,15 @@ namespace Catch { return m_nonHiddenFunctions; } + struct LexSort { + bool operator() (TestCase i,TestCase j) { return (i& matchingTestCases ) const { + struct RandomNumberGenerator { + int operator()( int n ) { return std::rand() % n; } + }; + for( std::vector::const_iterator it = m_functionsInOrder.begin(), itEnd = m_functionsInOrder.end(); it != itEnd; @@ -67,6 +75,17 @@ namespace Catch { if( testSpec.matches( *it ) && ( config.allowThrows() || !it->throws() ) ) matchingTestCases.push_back( *it ); } + switch( config.runOrder() ) { + case RunTests::InLexicographicalOrder: + std::sort( matchingTestCases.begin(), matchingTestCases.end(), LexSort() ); + break; + case RunTests::InRandomOrder: + std::random_shuffle( matchingTestCases.begin(), matchingTestCases.end(), RandomNumberGenerator() ); + break; + case RunTests::InDeclarationOrder: + // already in declaration order + break; + } } private: diff --git a/include/reporters/catch_reporter_console.hpp b/include/reporters/catch_reporter_console.hpp index 83498327..7c0a23bf 100644 --- a/include/reporters/catch_reporter_console.hpp +++ b/include/reporters/catch_reporter_console.hpp @@ -271,6 +271,9 @@ namespace Catch { stream << " host application.\n" << "Run with -? for options\n\n"; + if( m_config->rngSeed() != 0 ) + stream << "Randomness seeded to: " << m_config->rngSeed() << "\n\n"; + currentTestRunInfo.used = true; } void lazyPrintGroupInfo() {