SDL_test_harness.h File Reference

#include "begin_code.h"
#include "close_code.h"

Include dependency graph for SDL_test_harness.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures
struct	SDLTest_TestCaseReference
struct	SDLTest_TestSuiteReference

Macros
#define	TEST_ENABLED   1
#define	TEST_DISABLED   0
#define	TEST_ABORTED   -1
#define	TEST_STARTED   0
#define	TEST_COMPLETED   1
#define	TEST_SKIPPED   2
#define	TEST_RESULT_PASSED   0
#define	TEST_RESULT_FAILED   1
#define	TEST_RESULT_NO_ASSERT   2
#define	TEST_RESULT_SKIPPED   3
#define	TEST_RESULT_SETUP_FAILURE   4

Typedefs
typedef void(*	SDLTest_TestCaseSetUpFp) (void *arg)
typedef int(*	SDLTest_TestCaseFp) (void *arg)
typedef void(*	SDLTest_TestCaseTearDownFp) (void *arg)

Functions
char *	SDLTest_GenerateRunSeed (const int length)
	Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z). More...
int	SDLTest_RunSuites (SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
	Execute a test suite using the given run seed and execution key. More...

Detailed Description

Include file for SDL test framework.

This code is a part of the SDL2_test library, not the main SDL library.

Definition in file SDL_test_harness.h.

Macro Definition Documentation

TEST_ABORTED

#define TEST_ABORTED   -1

Definition at line 51 of file SDL_test_harness.h.

TEST_COMPLETED

#define TEST_COMPLETED   1

Definition at line 53 of file SDL_test_harness.h.

TEST_DISABLED

#define TEST_DISABLED   0

Definition at line 48 of file SDL_test_harness.h.

TEST_ENABLED

#define TEST_ENABLED   1

Definition at line 47 of file SDL_test_harness.h.

TEST_RESULT_FAILED

#define TEST_RESULT_FAILED   1

Definition at line 58 of file SDL_test_harness.h.

TEST_RESULT_NO_ASSERT

#define TEST_RESULT_NO_ASSERT   2

Definition at line 59 of file SDL_test_harness.h.

TEST_RESULT_PASSED

#define TEST_RESULT_PASSED   0

Definition at line 57 of file SDL_test_harness.h.

TEST_RESULT_SETUP_FAILURE

#define TEST_RESULT_SETUP_FAILURE   4

Definition at line 61 of file SDL_test_harness.h.

TEST_RESULT_SKIPPED

#define TEST_RESULT_SKIPPED   3

Definition at line 60 of file SDL_test_harness.h.

TEST_SKIPPED

#define TEST_SKIPPED   2

Definition at line 54 of file SDL_test_harness.h.

TEST_STARTED

#define TEST_STARTED   0

Definition at line 52 of file SDL_test_harness.h.

Typedef Documentation

SDLTest_TestCaseFp

typedef int(* SDLTest_TestCaseFp) (void *arg)

Definition at line 67 of file SDL_test_harness.h.

SDLTest_TestCaseSetUpFp

typedef void(* SDLTest_TestCaseSetUpFp) (void *arg)

Definition at line 64 of file SDL_test_harness.h.

SDLTest_TestCaseTearDownFp

typedef void(* SDLTest_TestCaseTearDownFp) (void *arg)

Definition at line 70 of file SDL_test_harness.h.

Function Documentation

SDLTest_GenerateRunSeed()

char* SDLTest_GenerateRunSeed

(

const int

length

)

Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z).

Note: The returned string needs to be deallocated by the caller.

Parameters

length

The length of the seed string to generate

Returns

The generated seed string

Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z).

Note: The returned string needs to be deallocated by the caller.

Parameters

length

The length of the seed string to generate

Returns

The generated seed string

Definition at line 54 of file SDL_test_harness.c.

{
    char *seed = NULL;
    SDLTest_RandomContext randomContext;
    int counter;
 
    /* Sanity check input */
    if (length <= 0) {
        SDLTest_LogError("The length of the harness seed must be >0.");
        return NULL;
    }
 
    /* Allocate output buffer */
    seed = (char *)SDL_malloc((length + 1) * sizeof(char));
    if (seed == NULL) {
        SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
        SDL_Error(SDL_ENOMEM);
        return NULL;
    }
 
    /* Generate a random string of alphanumeric characters */
    SDLTest_RandomInitTime(&randomContext);
    for (counter = 0; counter < length; counter++) {
        unsigned int number = SDLTest_Random(&randomContext);
        char ch = (char) (number % (91 - 48)) + 48;
        if (ch >= 58 && ch <= 64) {
            ch = 65;
        }
        seed[counter] = ch;
    }
    seed[length] = '\0';
 
    return seed;
}

SDLTest_RunSuites()

int SDLTest_RunSuites	(	SDLTest_TestSuiteReference *	testSuites[],
		const char *	userRunSeed,
		Uint64	userExecKey,
		const char *	filter,
		int	testIterations
	)

Execute a test suite using the given run seed and execution key.

Parameters

testSuites	Suites containing the test case.
userRunSeed	Custom run seed provided by user, or NULL to autogenerate one.
userExecKey	Custom execution key provided by user, or 0 to autogenerate one.
filter	Filter specification. NULL disables. Case sensitive.
testIterations	Number of iterations to run each test case.

Returns

Test run result; 0 when all tests passed, 1 if any tests failed.

The filter string is matched to the suite name (full comparison) to select a single suite, or if no suite matches, it is matched to the test names (full comparison) to select a single test.

Parameters

testSuites	Suites containing the test case.
userRunSeed	Custom run seed provided by user, or NULL to autogenerate one.
userExecKey	Custom execution key provided by user, or 0 to autogenerate one.
filter	Filter specification. NULL disables. Case sensitive.
testIterations	Number of iterations to run each test case.

Returns

Test run result; 0 when all tests passed, 1 if any tests failed.

Definition at line 370 of file SDL_test_harness.c.

{
    int totalNumberOfTests = 0;
    int failedNumberOfTests = 0;
    int suiteCounter;
    int testCounter;
    int iterationCounter;
    SDLTest_TestSuiteReference *testSuite;
    const SDLTest_TestCaseReference *testCase;
    const char *runSeed = NULL;
    char *currentSuiteName;
    char *currentTestName;
    Uint64 execKey;
    float runStartSeconds;
    float suiteStartSeconds;
    float testStartSeconds;
    float runEndSeconds;
    float suiteEndSeconds;
    float testEndSeconds;
    float runtime;
    int suiteFilter = 0;
    char *suiteFilterName = NULL;
    int testFilter = 0;
    char *testFilterName = NULL;
        SDL_bool forceTestRun = SDL_FALSE;
    int testResult = 0;
    int runResult = 0;
    Uint32 totalTestFailedCount = 0;
    Uint32 totalTestPassedCount = 0;
    Uint32 totalTestSkippedCount = 0;
    Uint32 testFailedCount = 0;
    Uint32 testPassedCount = 0;
    Uint32 testSkippedCount = 0;
    Uint32 countSum = 0;
    const SDLTest_TestCaseReference **failedTests;
 
    /* Sanitize test iterations */
    if (testIterations < 1) {
        testIterations = 1;
    }
 
    /* Generate run see if we don't have one already */
    if (userRunSeed == NULL || userRunSeed[0] == '\0') {
        runSeed = SDLTest_GenerateRunSeed(16);
        if (runSeed == NULL) {
            SDLTest_LogError("Generating a random seed failed");
            return 2;
        }
    } else {
        runSeed = userRunSeed;
    }
 
 
    /* Reset per-run counters */
    totalTestFailedCount = 0;
    totalTestPassedCount = 0;
    totalTestSkippedCount = 0;
 
    /* Take time - run start */
    runStartSeconds = GetClock();
 
    /* Log run with fuzzer parameters */
    SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
 
        /* Count the total number of tests */
    suiteCounter = 0;
    while (testSuites[suiteCounter]) {
        testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
        suiteCounter++;
        testCounter = 0;
        while (testSuite->testCases[testCounter])
        {
            testCounter++;
                        totalNumberOfTests++;
                }
        }
 
        /* Pre-allocate an array for tracking failed tests (potentially all test cases) */
        failedTests = (const SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
        if (failedTests == NULL) {      
           SDLTest_LogError("Unable to allocate cache for failed tests");
           SDL_Error(SDL_ENOMEM);          
           return -1;
        }
 
    /* Initialize filtering */
    if (filter != NULL && filter[0] != '\0') {
        /* Loop over all suites to check if we have a filter match */
        suiteCounter = 0;
        while (testSuites[suiteCounter] && suiteFilter == 0) {
            testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
            suiteCounter++;
            if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
                /* Matched a suite name */
                suiteFilter = 1;
                suiteFilterName = testSuite->name;
                SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
                break;
            }
 
            /* Within each suite, loop over all test cases to check if we have a filter match */
            testCounter = 0;
            while (testSuite->testCases[testCounter] && testFilter == 0)
            {
                testCase = testSuite->testCases[testCounter];
                testCounter++;
                if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
                    /* Matched a test name */
                    suiteFilter = 1;
                    suiteFilterName = testSuite->name;
                    testFilter = 1;
                    testFilterName = testCase->name;
                    SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);
                    break;
                }
            }
        }
 
        if (suiteFilter == 0 && testFilter == 0) {
            SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
            SDLTest_Log("Exit code: 2");
            SDL_free((void *) failedTests);
            return 2;
        }
    }
 
    /* Loop over all suites */
    suiteCounter = 0;
    while(testSuites[suiteCounter]) {
        testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
        currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
        suiteCounter++;
 
        /* Filter suite if flag set and we have a name */
        if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
            SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
                /* Skip suite */
                SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
                    suiteCounter,
                    currentSuiteName);
        } else {
 
            /* Reset per-suite counters */
            testFailedCount = 0;
            testPassedCount = 0;
            testSkippedCount = 0;
 
            /* Take time - suite start */
            suiteStartSeconds = GetClock();
 
            /* Log suite started */
            SDLTest_Log("===== Test Suite %i: '%s' started\n",
                suiteCounter,
                currentSuiteName);
 
            /* Loop over all test cases */
            testCounter = 0;
            while(testSuite->testCases[testCounter])
            {
                testCase = testSuite->testCases[testCounter];
                currentTestName = (char *)((testCase->name) ? testCase->name : SDLTEST_INVALID_NAME_FORMAT);
                testCounter++;
 
                /* Filter tests if flag set and we have a name */
                if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
                    SDL_strcmp(testFilterName, testCase->name) != 0) {
                        /* Skip test */
                        SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
                            suiteCounter,
                            testCounter,
                            currentTestName);
                } else {
                    /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
                    if (testFilter == 1 && !testCase->enabled) {
                        SDLTest_Log("Force run of disabled test since test filter was set");
                                                forceTestRun = SDL_TRUE;
                    }
 
                    /* Take time - test start */
                    testStartSeconds = GetClock();
 
                    /* Log test started */
                    SDLTest_Log("----- Test Case %i.%i: '%s' started",
                        suiteCounter,
                        testCounter,
                        currentTestName);
                    if (testCase->description != NULL && testCase->description[0] != '\0') {
                        SDLTest_Log("Test Description: '%s'",
                            (testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT);
                    }
 
                    /* Loop over all iterations */
                    iterationCounter = 0;
                    while(iterationCounter < testIterations)
                    {
                        iterationCounter++;
 
                        if (userExecKey != 0) {
                            execKey = userExecKey;
                        } else {
                            execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
                        }
 
                        SDLTest_Log("Test Iteration %i: execKey %" SDL_PRIu64, iterationCounter, execKey);
                                                testResult = SDLTest_RunTest(testSuite, testCase, execKey, forceTestRun);
 
                        if (testResult == TEST_RESULT_PASSED) {
                            testPassedCount++;
                            totalTestPassedCount++;
                        } else if (testResult == TEST_RESULT_SKIPPED) {
                            testSkippedCount++;
                            totalTestSkippedCount++;
                        } else {
                            testFailedCount++;
                            totalTestFailedCount++;
                        }
                    }
 
                    /* Take time - test end */
                    testEndSeconds = GetClock();
                    runtime = testEndSeconds - testStartSeconds;
                    if (runtime < 0.0f) runtime = 0.0f;
 
                    if (testIterations > 1) {
                        /* Log test runtime */
                        SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
                        SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
                    } else {
                        /* Log test runtime */
                        SDLTest_Log("Total Test runtime: %.1f sec", runtime);
                    }
 
                    /* Log final test result */
                    switch (testResult) {
                    case TEST_RESULT_PASSED:
                        SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Passed");
                        break;
                    case TEST_RESULT_FAILED:
                        SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Failed");
                        break;
                    case TEST_RESULT_NO_ASSERT:
                        SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT,"Test", currentTestName, "No Asserts");
                        break;
                    }
 
                    /* Collect failed test case references for repro-step display */
                    if (testResult == TEST_RESULT_FAILED) {
                        failedTests[failedNumberOfTests] = testCase;
                        failedNumberOfTests++;
                    }
                }
            }
 
            /* Take time - suite end */
            suiteEndSeconds = GetClock();
            runtime = suiteEndSeconds - suiteStartSeconds;
            if (runtime < 0.0f) runtime = 0.0f;
 
            /* Log suite runtime */
            SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
 
            /* Log summary and final Suite result */
            countSum = testPassedCount + testFailedCount + testSkippedCount;
            if (testFailedCount == 0)
            {
                SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
                SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Passed");
            }
            else
            {
                SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
                SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Failed");
            }
 
        }
    }
 
    /* Take time - run end */
    runEndSeconds = GetClock();
    runtime = runEndSeconds - runStartSeconds;
    if (runtime < 0.0f) runtime = 0.0f;
 
    /* Log total runtime */
    SDLTest_Log("Total Run runtime: %.1f sec", runtime);
 
    /* Log summary and final run result */
    countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
    if (totalTestFailedCount == 0)
    {
        runResult = 0;
        SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
        SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Passed");
    }
    else
    {
        runResult = 1;
        SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
        SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Failed");
    }
 
    /* Print repro steps for failed tests */
    if (failedNumberOfTests > 0) {
        SDLTest_Log("Harness input to repro failures:");
        for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
          SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
        }
    }
    SDL_free((void *) failedTests);
 
    SDLTest_Log("Exit code: %d", runResult);
    return runResult;
}

References SDLTest_TestCaseReference::description, SDLTest_TestCaseReference::enabled, GetClock(), SDLTest_TestCaseReference::name, SDLTest_TestSuiteReference::name, NULL, SDL_ENOMEM, SDL_Error, SDL_FALSE, SDL_free, SDL_malloc, SDL_PRIu64, SDL_strcmp, SDL_TRUE, SDLTEST_FINAL_RESULT_FORMAT, SDLTest_GenerateExecKey(), SDLTest_GenerateRunSeed(), SDLTEST_INVALID_NAME_FORMAT, SDLTest_Log(), SDLTEST_LOG_SUMMARY_FORMAT, SDLTest_LogError(), SDLTest_RunTest(), TEST_RESULT_FAILED, TEST_RESULT_NO_ASSERT, TEST_RESULT_PASSED, TEST_RESULT_SKIPPED, SDLTest_TestSuiteReference::testCases, and testSuites.

Referenced by main().