SDL  2.0
SDL_test_harness.c File Reference
#include "SDL_config.h"
#include "SDL_test.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
+ Include dependency graph for SDL_test_harness.c:

Go to the source code of this file.

Macros

#define SDLTEST_INVALID_NAME_FORMAT   "(Invalid)"
 
#define SDLTEST_LOG_SUMMARY_FORMAT   "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d"
 
#define SDLTEST_FINAL_RESULT_FORMAT   ">>> %s '%s': %s\n"
 

Functions

char * SDLTest_GenerateRunSeed (const int length)
 
Uint64 SDLTest_GenerateExecKey (char *runSeed, char *suiteName, char *testName, int iteration)
 
SDL_TimerID SDLTest_SetTestTimeout (int timeout, void(*callback)())
 Set timeout handler for test. More...
 
void SDLTest_BailOut ()
 Timeout handler. Aborts test run and exits harness process. More...
 
int SDLTest_RunTest (SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
 Execute a test using the given execution key. More...
 
void SDLTest_LogTestSuiteSummary (SDLTest_TestSuiteReference *testSuites)
 
float GetClock ()
 
int SDLTest_RunSuites (SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
 Execute a test suite using the given run seed and execution key. More...
 

Variables

static Uint32 SDLTest_TestCaseTimeout = 3600
 

Macro Definition Documentation

◆ SDLTEST_FINAL_RESULT_FORMAT

#define SDLTEST_FINAL_RESULT_FORMAT   ">>> %s '%s': %s\n"

Definition at line 38 of file SDL_test_harness.c.

Referenced by SDLTest_RunSuites(), and SDLTest_RunTest().

◆ SDLTEST_INVALID_NAME_FORMAT

#define SDLTEST_INVALID_NAME_FORMAT   "(Invalid)"

Definition at line 32 of file SDL_test_harness.c.

Referenced by SDLTest_LogTestSuiteSummary(), and SDLTest_RunSuites().

◆ SDLTEST_LOG_SUMMARY_FORMAT

#define SDLTEST_LOG_SUMMARY_FORMAT   "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d"

Definition at line 35 of file SDL_test_harness.c.

Referenced by SDLTest_RunSuites().

Function Documentation

◆ GetClock()

float GetClock ( )

Definition at line 345 of file SDL_test_harness.c.

Referenced by SDLTest_RunSuites().

346 {
347  float currentClock = (float)clock();
348  return currentClock / (float)CLOCKS_PER_SEC;
349 }

◆ SDLTest_BailOut()

void SDLTest_BailOut ( )

Timeout handler. Aborts test run and exits harness process.

Definition at line 210 of file SDL_test_harness.c.

References SDLTest_LogError(), and TEST_ABORTED.

Referenced by SDLTest_RunTest().

211 {
212  SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
213  exit(TEST_ABORTED); /* bail out from the test */
214 }
#define TEST_ABORTED
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:88

◆ SDLTest_GenerateExecKey()

Uint64 SDLTest_GenerateExecKey ( char *  runSeed,
char *  suiteName,
char *  testName,
int  iteration 
)

Generates an execution key for the fuzzer.

Parameters
runSeedThe run seed to use
suiteNameThe name of the test suite
testNameThe name of the test
iterationThe iteration count
Returns
The generated execution key to initialize the fuzzer with.

Definition at line 101 of file SDL_test_harness.c.

References SDLTest_Md5Context::digest, NULL, SDL_ENOMEM, SDL_Error, SDL_free(), SDL_malloc, SDL_memset, SDL_snprintf, SDL_strlen, SDLTest_LogError(), SDLTest_Md5Final(), SDLTest_Md5Init(), and SDLTest_Md5Update().

Referenced by SDLTest_RunSuites().

102 {
103  SDLTest_Md5Context md5Context;
104  Uint64 *keys;
105  char iterationString[16];
106  Uint32 runSeedLength;
107  Uint32 suiteNameLength;
108  Uint32 testNameLength;
109  Uint32 iterationStringLength;
110  Uint32 entireStringLength;
111  char *buffer;
112 
113  if (runSeed == NULL || runSeed[0] == '\0') {
114  SDLTest_LogError("Invalid runSeed string.");
115  return -1;
116  }
117 
118  if (suiteName == NULL || suiteName[0] == '\0') {
119  SDLTest_LogError("Invalid suiteName string.");
120  return -1;
121  }
122 
123  if (testName == NULL || testName[0] == '\0') {
124  SDLTest_LogError("Invalid testName string.");
125  return -1;
126  }
127 
128  if (iteration <= 0) {
129  SDLTest_LogError("Invalid iteration count.");
130  return -1;
131  }
132 
133  /* Convert iteration number into a string */
134  SDL_memset(iterationString, 0, sizeof(iterationString));
135  SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
136 
137  /* Combine the parameters into single string */
138  runSeedLength = SDL_strlen(runSeed);
139  suiteNameLength = SDL_strlen(suiteName);
140  testNameLength = SDL_strlen(testName);
141  iterationStringLength = SDL_strlen(iterationString);
142  entireStringLength = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
143  buffer = (char *)SDL_malloc(entireStringLength);
144  if (buffer == NULL) {
145  SDLTest_LogError("Failed to allocate buffer for execKey generation.");
147  return 0;
148  }
149  SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
150 
151  /* Hash string and use half of the digest as 64bit exec key */
152  SDLTest_Md5Init(&md5Context);
153  SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, entireStringLength);
154  SDLTest_Md5Final(&md5Context);
155  SDL_free(buffer);
156  keys = (Uint64 *)md5Context.digest;
157 
158  return keys[0];
159 }
uint32_t Uint32
An unsigned 32-bit integer type.
Definition: SDL_stdinc.h:159
#define SDL_Error
uint64_t Uint64
An unsigned 64-bit integer type.
Definition: SDL_stdinc.h:168
static void iteration()
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:88
void SDL_free(void *mem)
void SDLTest_Md5Update(SDLTest_Md5Context *mdContext, unsigned char *inBuf, unsigned int inLen)
update digest from variable length data
Definition: SDL_test_md5.c:131
void SDLTest_Md5Init(SDLTest_Md5Context *mdContext)
initialize the context
Definition: SDL_test_md5.c:110
#define NULL
Definition: begin_code.h:143
#define SDL_strlen
void SDLTest_Md5Final(SDLTest_Md5Context *mdContext)
complete digest computation
Definition: SDL_test_md5.c:180
GLuint buffer
#define SDL_snprintf
#define SDL_malloc
#define SDL_memset
unsigned char digest[16]
Definition: SDL_test_md5.h:75

◆ SDLTest_GenerateRunSeed()

char* SDLTest_GenerateRunSeed ( const int  length)

Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z).

Note: The returned string needs to be deallocated by the caller.

Parameters
lengthThe length of the seed string to generate
Returns
The generated seed string

Definition at line 54 of file SDL_test_harness.c.

References NULL, SDL_ENOMEM, SDL_Error, SDL_malloc, SDLTest_LogError(), SDLTest_Random(), and SDLTest_RandomInitTime().

Referenced by sdltest_generateRunSeed(), and SDLTest_RunSuites().

55 {
56  char *seed = NULL;
57  SDLTest_RandomContext randomContext;
58  int counter;
59 
60  /* Sanity check input */
61  if (length <= 0) {
62  SDLTest_LogError("The length of the harness seed must be >0.");
63  return NULL;
64  }
65 
66  /* Allocate output buffer */
67  seed = (char *)SDL_malloc((length + 1) * sizeof(char));
68  if (seed == NULL) {
69  SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
71  return NULL;
72  }
73 
74  /* Generate a random string of alphanumeric characters */
75  SDLTest_RandomInitTime(&randomContext);
76  for (counter = 0; counter < length; counter++) {
77  unsigned int number = SDLTest_Random(&randomContext);
78  char ch = (char) (number % (91 - 48)) + 48;
79  if (ch >= 58 && ch <= 64) {
80  ch = 65;
81  }
82  seed[counter] = ch;
83  }
84  seed[length] = '\0';
85 
86  return seed;
87 }
#define SDL_Error
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:88
void SDLTest_RandomInitTime(SDLTest_RandomContext *rndContext)
Initialize random number generator based on current system time.
unsigned int SDLTest_Random(SDLTest_RandomContext *rndContext)
Initialize random number generator based on current system time.
#define NULL
Definition: begin_code.h:143
GLuint counter
#define SDL_malloc
GLuint GLsizei GLsizei * length

◆ SDLTest_LogTestSuiteSummary()

void SDLTest_LogTestSuiteSummary ( SDLTest_TestSuiteReference testSuites)

Definition at line 316 of file SDL_test_harness.c.

References SDLTest_TestCaseReference::description, SDLTest_TestCaseReference::name, SDLTest_TestSuiteReference::name, SDLTEST_INVALID_NAME_FORMAT, SDLTest_Log(), and SDLTest_TestSuiteReference::testCases.

317 {
318  int suiteCounter;
319  int testCounter;
320  SDLTest_TestSuiteReference *testSuite;
321  SDLTest_TestCaseReference *testCase;
322 
323  /* Loop over all suites */
324  suiteCounter = 0;
325  while(&testSuites[suiteCounter]) {
326  testSuite=&testSuites[suiteCounter];
327  suiteCounter++;
328  SDLTest_Log("Test Suite %i - %s\n", suiteCounter,
329  (testSuite->name) ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
330 
331  /* Loop over all test cases */
332  testCounter = 0;
333  while(testSuite->testCases[testCounter])
334  {
335  testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
336  testCounter++;
337  SDLTest_Log(" Test Case %i - %s: %s", testCounter,
338  (testCase->name) ? testCase->name : SDLTEST_INVALID_NAME_FORMAT,
339  (testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT);
340  }
341  }
342 }
#define SDLTEST_INVALID_NAME_FORMAT
const SDLTest_TestCaseReference ** testCases
void SDLTest_Log(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and INFO priority.
Definition: SDL_test_log.c:70

◆ SDLTest_RunSuites()

int SDLTest_RunSuites ( SDLTest_TestSuiteReference testSuites[],
const char *  userRunSeed,
Uint64  userExecKey,
const char *  filter,
int  testIterations 
)

Execute a test suite using the given run seed and execution key.

The filter string is matched to the suite name (full comparison) to select a single suite, or if no suite matches, it is matched to the test names (full comparison) to select a single test.

Parameters
testSuitesSuites containing the test case.
userRunSeedCustom run seed provided by user, or NULL to autogenerate one.
userExecKeyCustom execution key provided by user, or 0 to autogenerate one.
filterFilter specification. NULL disables. Case sensitive.
testIterationsNumber of iterations to run each test case.
Returns
Test run result; 0 when all tests passed, 1 if any tests failed.

Definition at line 365 of file SDL_test_harness.c.

References SDLTest_TestCaseReference::description, SDLTest_TestCaseReference::enabled, GetClock(), SDLTest_TestCaseReference::name, SDLTest_TestSuiteReference::name, NULL, SDL_ENOMEM, SDL_Error, SDL_FALSE, SDL_free(), SDL_malloc, SDL_PRIu64, SDL_strcmp, SDL_TRUE, SDLTEST_FINAL_RESULT_FORMAT, SDLTest_GenerateExecKey(), SDLTest_GenerateRunSeed(), SDLTEST_INVALID_NAME_FORMAT, SDLTest_Log(), SDLTEST_LOG_SUMMARY_FORMAT, SDLTest_LogError(), SDLTest_RunTest(), TEST_RESULT_FAILED, TEST_RESULT_NO_ASSERT, TEST_RESULT_PASSED, TEST_RESULT_SKIPPED, and SDLTest_TestSuiteReference::testCases.

Referenced by main().

366 {
367  int totalNumberOfTests = 0;
368  int failedNumberOfTests = 0;
369  int suiteCounter;
370  int testCounter;
371  int iterationCounter;
372  SDLTest_TestSuiteReference *testSuite;
373  SDLTest_TestCaseReference *testCase;
374  const char *runSeed = NULL;
375  char *currentSuiteName;
376  char *currentTestName;
377  Uint64 execKey;
378  float runStartSeconds;
379  float suiteStartSeconds;
380  float testStartSeconds;
381  float runEndSeconds;
382  float suiteEndSeconds;
383  float testEndSeconds;
384  float runtime;
385  int suiteFilter = 0;
386  char *suiteFilterName = NULL;
387  int testFilter = 0;
388  char *testFilterName = NULL;
389  SDL_bool forceTestRun = SDL_FALSE;
390  int testResult = 0;
391  int runResult = 0;
392  Uint32 totalTestFailedCount = 0;
393  Uint32 totalTestPassedCount = 0;
394  Uint32 totalTestSkippedCount = 0;
395  Uint32 testFailedCount = 0;
396  Uint32 testPassedCount = 0;
397  Uint32 testSkippedCount = 0;
398  Uint32 countSum = 0;
399  SDLTest_TestCaseReference **failedTests;
400 
401  /* Sanitize test iterations */
402  if (testIterations < 1) {
403  testIterations = 1;
404  }
405 
406  /* Generate run see if we don't have one already */
407  if (userRunSeed == NULL || userRunSeed[0] == '\0') {
408  runSeed = SDLTest_GenerateRunSeed(16);
409  if (runSeed == NULL) {
410  SDLTest_LogError("Generating a random seed failed");
411  return 2;
412  }
413  } else {
414  runSeed = userRunSeed;
415  }
416 
417 
418  /* Reset per-run counters */
419  totalTestFailedCount = 0;
420  totalTestPassedCount = 0;
421  totalTestSkippedCount = 0;
422 
423  /* Take time - run start */
424  runStartSeconds = GetClock();
425 
426  /* Log run with fuzzer parameters */
427  SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
428 
429  /* Count the total number of tests */
430  suiteCounter = 0;
431  while (testSuites[suiteCounter]) {
432  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
433  suiteCounter++;
434  testCounter = 0;
435  while (testSuite->testCases[testCounter])
436  {
437  testCounter++;
438  totalNumberOfTests++;
439  }
440  }
441 
442  /* Pre-allocate an array for tracking failed tests (potentially all test cases) */
443  failedTests = (SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
444  if (failedTests == NULL) {
445  SDLTest_LogError("Unable to allocate cache for failed tests");
447  return -1;
448  }
449 
450  /* Initialize filtering */
451  if (filter != NULL && filter[0] != '\0') {
452  /* Loop over all suites to check if we have a filter match */
453  suiteCounter = 0;
454  while (testSuites[suiteCounter] && suiteFilter == 0) {
455  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
456  suiteCounter++;
457  if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
458  /* Matched a suite name */
459  suiteFilter = 1;
460  suiteFilterName = testSuite->name;
461  SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
462  break;
463  }
464 
465  /* Within each suite, loop over all test cases to check if we have a filter match */
466  testCounter = 0;
467  while (testSuite->testCases[testCounter] && testFilter == 0)
468  {
469  testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
470  testCounter++;
471  if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
472  /* Matched a test name */
473  suiteFilter = 1;
474  suiteFilterName = testSuite->name;
475  testFilter = 1;
476  testFilterName = testCase->name;
477  SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);
478  break;
479  }
480  }
481  }
482 
483  if (suiteFilter == 0 && testFilter == 0) {
484  SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
485  SDLTest_Log("Exit code: 2");
486  SDL_free(failedTests);
487  return 2;
488  }
489  }
490 
491  /* Loop over all suites */
492  suiteCounter = 0;
493  while(testSuites[suiteCounter]) {
494  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
495  currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
496  suiteCounter++;
497 
498  /* Filter suite if flag set and we have a name */
499  if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
500  SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
501  /* Skip suite */
502  SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
503  suiteCounter,
504  currentSuiteName);
505  } else {
506 
507  /* Reset per-suite counters */
508  testFailedCount = 0;
509  testPassedCount = 0;
510  testSkippedCount = 0;
511 
512  /* Take time - suite start */
513  suiteStartSeconds = GetClock();
514 
515  /* Log suite started */
516  SDLTest_Log("===== Test Suite %i: '%s' started\n",
517  suiteCounter,
518  currentSuiteName);
519 
520  /* Loop over all test cases */
521  testCounter = 0;
522  while(testSuite->testCases[testCounter])
523  {
524  testCase=(SDLTest_TestCaseReference *)testSuite->testCases[testCounter];
525  currentTestName = (char *)((testCase->name) ? testCase->name : SDLTEST_INVALID_NAME_FORMAT);
526  testCounter++;
527 
528  /* Filter tests if flag set and we have a name */
529  if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
530  SDL_strcmp(testFilterName, testCase->name) != 0) {
531  /* Skip test */
532  SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
533  suiteCounter,
534  testCounter,
535  currentTestName);
536  } else {
537  /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
538  if (testFilter == 1 && !testCase->enabled) {
539  SDLTest_Log("Force run of disabled test since test filter was set");
540  forceTestRun = SDL_TRUE;
541  }
542 
543  /* Take time - test start */
544  testStartSeconds = GetClock();
545 
546  /* Log test started */
547  SDLTest_Log("----- Test Case %i.%i: '%s' started",
548  suiteCounter,
549  testCounter,
550  currentTestName);
551  if (testCase->description != NULL && testCase->description[0] != '\0') {
552  SDLTest_Log("Test Description: '%s'",
553  (testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT);
554  }
555 
556  /* Loop over all iterations */
557  iterationCounter = 0;
558  while(iterationCounter < testIterations)
559  {
560  iterationCounter++;
561 
562  if (userExecKey != 0) {
563  execKey = userExecKey;
564  } else {
565  execKey = SDLTest_GenerateExecKey((char *)runSeed, testSuite->name, testCase->name, iterationCounter);
566  }
567 
568  SDLTest_Log("Test Iteration %i: execKey %" SDL_PRIu64, iterationCounter, execKey);
569  testResult = SDLTest_RunTest(testSuite, testCase, execKey, forceTestRun);
570 
571  if (testResult == TEST_RESULT_PASSED) {
572  testPassedCount++;
573  totalTestPassedCount++;
574  } else if (testResult == TEST_RESULT_SKIPPED) {
575  testSkippedCount++;
576  totalTestSkippedCount++;
577  } else {
578  testFailedCount++;
579  totalTestFailedCount++;
580  }
581  }
582 
583  /* Take time - test end */
584  testEndSeconds = GetClock();
585  runtime = testEndSeconds - testStartSeconds;
586  if (runtime < 0.0f) runtime = 0.0f;
587 
588  if (testIterations > 1) {
589  /* Log test runtime */
590  SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
591  SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
592  } else {
593  /* Log test runtime */
594  SDLTest_Log("Total Test runtime: %.1f sec", runtime);
595  }
596 
597  /* Log final test result */
598  switch (testResult) {
599  case TEST_RESULT_PASSED:
600  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Passed");
601  break;
602  case TEST_RESULT_FAILED:
603  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Failed");
604  break;
606  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT,"Test", currentTestName, "No Asserts");
607  break;
608  }
609 
610  /* Collect failed test case references for repro-step display */
611  if (testResult == TEST_RESULT_FAILED) {
612  failedTests[failedNumberOfTests] = testCase;
613  failedNumberOfTests++;
614  }
615  }
616  }
617 
618  /* Take time - suite end */
619  suiteEndSeconds = GetClock();
620  runtime = suiteEndSeconds - suiteStartSeconds;
621  if (runtime < 0.0f) runtime = 0.0f;
622 
623  /* Log suite runtime */
624  SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
625 
626  /* Log summary and final Suite result */
627  countSum = testPassedCount + testFailedCount + testSkippedCount;
628  if (testFailedCount == 0)
629  {
630  SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
631  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Passed");
632  }
633  else
634  {
635  SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
636  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Failed");
637  }
638 
639  }
640  }
641 
642  /* Take time - run end */
643  runEndSeconds = GetClock();
644  runtime = runEndSeconds - runStartSeconds;
645  if (runtime < 0.0f) runtime = 0.0f;
646 
647  /* Log total runtime */
648  SDLTest_Log("Total Run runtime: %.1f sec", runtime);
649 
650  /* Log summary and final run result */
651  countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
652  if (totalTestFailedCount == 0)
653  {
654  runResult = 0;
655  SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
656  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Passed");
657  }
658  else
659  {
660  runResult = 1;
661  SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
662  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Failed");
663  }
664 
665  /* Print repro steps for failed tests */
666  if (failedNumberOfTests > 0) {
667  SDLTest_Log("Harness input to repro failures:");
668  for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
669  SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
670  }
671  }
672  SDL_free(failedTests);
673 
674  SDLTest_Log("Exit code: %d", runResult);
675  return runResult;
676 }
#define SDLTEST_INVALID_NAME_FORMAT
char * SDLTest_GenerateRunSeed(const int length)
#define SDLTEST_FINAL_RESULT_FORMAT
GLfloat f
float GetClock()
GLuint const GLchar * name
uint32_t Uint32
An unsigned 32-bit integer type.
Definition: SDL_stdinc.h:159
#define SDL_Error
uint64_t Uint64
An unsigned 64-bit integer type.
Definition: SDL_stdinc.h:168
const SDLTest_TestCaseReference ** testCases
Uint64 SDLTest_GenerateExecKey(char *runSeed, char *suiteName, char *testName, int iteration)
#define TEST_RESULT_FAILED
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:88
void SDL_free(void *mem)
#define TEST_RESULT_NO_ASSERT
#define NULL
Definition: begin_code.h:143
SDL_bool
Definition: SDL_stdinc.h:130
#define TEST_RESULT_PASSED
#define SDLTEST_LOG_SUMMARY_FORMAT
#define SDL_PRIu64
Definition: SDL_stdinc.h:190
int SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
Execute a test using the given execution key.
void SDLTest_Log(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and INFO priority.
Definition: SDL_test_log.c:70
#define SDL_malloc
#define SDL_strcmp
#define TEST_RESULT_SKIPPED
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter

◆ SDLTest_RunTest()

int SDLTest_RunTest ( SDLTest_TestSuiteReference testSuite,
SDLTest_TestCaseReference testCase,
Uint64  execKey,
SDL_bool  forceTestRun 
)

Execute a test using the given execution key.

Parameters
testSuiteSuite containing the test case.
testCaseCase to execute.
execKeyExecution key for the fuzzer.
forceTestRunForce test to run even if test was disabled in suite.
Returns
Test case result.

Definition at line 227 of file SDL_test_harness.c.

References SDLTest_TestCaseReference::enabled, SDLTest_TestCaseReference::name, SDLTest_TestSuiteReference::name, NULL, SDL_FALSE, SDL_RemoveTimer, SDLTest_AssertSummaryToTestResult(), SDLTest_BailOut(), SDLTEST_FINAL_RESULT_FORMAT, SDLTest_FuzzerInit(), SDLTest_GetFuzzerInvocationCount(), SDLTest_Log(), SDLTest_LogAssertSummary(), SDLTest_LogError(), SDLTest_ResetAssertSummary(), SDLTest_SetTestTimeout(), SDLTest_TestCaseTimeout, TEST_ABORTED, TEST_RESULT_FAILED, TEST_RESULT_SETUP_FAILURE, TEST_RESULT_SKIPPED, TEST_SKIPPED, TEST_STARTED, SDLTest_TestCaseReference::testCase, SDLTest_TestSuiteReference::testSetUp, and SDLTest_TestSuiteReference::testTearDown.

Referenced by SDLTest_RunSuites().

228 {
229  SDL_TimerID timer = 0;
230  int testCaseResult = 0;
231  int testResult = 0;
232  int fuzzerCount;
233 
234  if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
235  {
236  SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
238  }
239 
240  if (!testCase->enabled && forceTestRun == SDL_FALSE)
241  {
242  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Disabled)");
243  return TEST_RESULT_SKIPPED;
244  }
245 
246  /* Initialize fuzzer */
247  SDLTest_FuzzerInit(execKey);
248 
249  /* Reset assert tracker */
251 
252  /* Set timeout timer */
254 
255  /* Maybe run suite initalizer function */
256  if (testSuite->testSetUp) {
257  testSuite->testSetUp(0x0);
259  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite Setup", testSuite->name, "Failed");
261  }
262  }
263 
264  /* Run test case function */
265  testCaseResult = testCase->testCase(0x0);
266 
267  /* Convert test execution result into harness result */
268  if (testCaseResult == TEST_SKIPPED) {
269  /* Test was programatically skipped */
270  testResult = TEST_RESULT_SKIPPED;
271  } else if (testCaseResult == TEST_STARTED) {
272  /* Test did not return a TEST_COMPLETED value; assume it failed */
273  testResult = TEST_RESULT_FAILED;
274  } else if (testCaseResult == TEST_ABORTED) {
275  /* Test was aborted early; assume it failed */
276  testResult = TEST_RESULT_FAILED;
277  } else {
278  /* Perform failure analysis based on asserts */
279  testResult = SDLTest_AssertSummaryToTestResult();
280  }
281 
282  /* Maybe run suite cleanup function (ignore failed asserts) */
283  if (testSuite->testTearDown) {
284  testSuite->testTearDown(0x0);
285  }
286 
287  /* Cancel timeout timer */
288  if (timer) {
289  SDL_RemoveTimer(timer);
290  }
291 
292  /* Report on asserts and fuzzer usage */
293  fuzzerCount = SDLTest_GetFuzzerInvocationCount();
294  if (fuzzerCount > 0) {
295  SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
296  }
297 
298  /* Final log based on test execution result */
299  if (testCaseResult == TEST_SKIPPED) {
300  /* Test was programatically skipped */
301  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Programmatically)");
302  } else if (testCaseResult == TEST_STARTED) {
303  /* Test did not return a TEST_COMPLETED value; assume it failed */
304  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
305  } else if (testCaseResult == TEST_ABORTED) {
306  /* Test was aborted early; assume it failed */
307  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (Aborted)");
308  } else {
310  }
311 
312  return testResult;
313 }
#define TEST_ABORTED
SDLTest_TestCaseTearDownFp testTearDown
#define SDLTEST_FINAL_RESULT_FORMAT
static Uint32 SDLTest_TestCaseTimeout
void SDLTest_BailOut()
Timeout handler. Aborts test run and exits harness process.
SDLTest_TestCaseSetUpFp testSetUp
#define TEST_RESULT_FAILED
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:88
SDL_TimerID SDLTest_SetTestTimeout(int timeout, void(*callback)())
Set timeout handler for test.
#define TEST_RESULT_SETUP_FAILURE
#define TEST_STARTED
int SDLTest_AssertSummaryToTestResult()
Converts the current assert summary state to a test result.
#define NULL
Definition: begin_code.h:143
int SDLTest_GetFuzzerInvocationCount()
void SDLTest_ResetAssertSummary()
Resets the assert summary counters to zero.
GLuint GLfloat x0
void SDLTest_Log(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and INFO priority.
Definition: SDL_test_log.c:70
void SDLTest_FuzzerInit(Uint64 execKey)
#define TEST_RESULT_SKIPPED
SDLTest_TestCaseFp testCase
#define TEST_SKIPPED
void SDLTest_LogAssertSummary()
Logs summary of all assertions (total, pass, fail) since last reset as INFO or ERROR.
#define SDL_RemoveTimer
int SDL_TimerID
Definition: SDL_timer.h:86

◆ SDLTest_SetTestTimeout()

SDL_TimerID SDLTest_SetTestTimeout ( int  timeout,
void(*)()  callback 
)

Set timeout handler for test.

Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.

Parameters
timeoutTimeout interval in seconds.
callbackFunction that will be called after timeout has elapsed.
Returns
Timer id or -1 on failure.

Definition at line 172 of file SDL_test_harness.c.

References callback(), NULL, SDL_AddTimer, SDL_GetError, SDL_INIT_TIMER, SDL_InitSubSystem, SDL_WasInit, and SDLTest_LogError().

Referenced by SDLTest_RunTest().

173 {
174  Uint32 timeoutInMilliseconds;
175  SDL_TimerID timerID;
176 
177  if (callback == NULL) {
178  SDLTest_LogError("Timeout callback can't be NULL");
179  return -1;
180  }
181 
182  if (timeout < 0) {
183  SDLTest_LogError("Timeout value must be bigger than zero.");
184  return -1;
185  }
186 
187  /* Init SDL timer if not initialized before */
188  if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
190  SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
191  return -1;
192  }
193  }
194 
195  /* Set timer */
196  timeoutInMilliseconds = timeout * 1000;
197  timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
198  if (timerID == 0) {
199  SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
200  return -1;
201  }
202 
203  return timerID;
204 }
#define SDL_GetError
Uint32(* SDL_TimerCallback)(Uint32 interval, void *param)
Definition: SDL_timer.h:81
#define SDL_InitSubSystem
uint32_t Uint32
An unsigned 32-bit integer type.
Definition: SDL_stdinc.h:159
#define SDL_AddTimer
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:88
static Uint32 callback(Uint32 interval, void *param)
Definition: testtimer.c:34
#define NULL
Definition: begin_code.h:143
#define SDL_INIT_TIMER
Definition: SDL.h:75
GLbitfield GLuint64 timeout
GLuint GLfloat x0
#define SDL_WasInit
int SDL_TimerID
Definition: SDL_timer.h:86

Variable Documentation

◆ SDLTest_TestCaseTimeout

Uint32 SDLTest_TestCaseTimeout = 3600
static

Definition at line 41 of file SDL_test_harness.c.

Referenced by SDLTest_RunTest().