From 7bf9e7cd3814f336e3305843017f9a584587be4b Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 17 Jan 2025 12:53:04 +0000 Subject: [PATCH 01/10] allow customization for app and test prefixes Allow the resulting application names to be configured by the user, instead of hardcoding `clar_suite.h` and `clar.suite`. This configuration will also customize the struct names (`clar_func`, etc.) Also allow the test names to be configured by the user, instead of hardcoding `test_` as a prefix. This allows users to generate test functions with uniquely prefixed names, for example, if they were generating benchmark code instead of unit tests. --- generate.py | 58 +++++++++++++++++++++++++++++------------------------ 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/generate.py b/generate.py index 2357b2d..7f40979 100755 --- a/generate.py +++ b/generate.py @@ -34,7 +34,7 @@ def render(self): class CallbacksTemplate(Template): def render(self): - out = "static const struct clar_func _clar_cb_%s[] = {\n" % self.module.name + out = "static const struct %s_func _%s_cb_%s[] = {\n" % (self.module.app_name, self.module.app_name, self.module.name) out += ",\n".join(self._render_callback(cb) for cb in self.module.callbacks) out += "\n};\n" return out @@ -65,7 +65,7 @@ def render(self): clean_name = name, initialize = self._render_callback(initializer), cleanup = self._render_callback(self.module.cleanup), - cb_ptr = "_clar_cb_%s" % self.module.name, + cb_ptr = "_%s_cb_%s" % (self.module.app_name, self.module.name), cb_count = len(self.module.callbacks), enabled = int(self.module.enabled) ) @@ -73,8 +73,10 @@ def render(self): return ','.join(templates) - def __init__(self, name): + def __init__(self, name, app_name, prefix): self.name = name + self.app_name = app_name + self.prefix = prefix self.mtime = None self.enabled = True @@ -179,8 +181,8 @@ def find_modules(self): return modules - def load_cache(self): - path = os.path.join(self.output, '.clarcache') + def load_cache(self, app_name): + path = os.path.join(self.output, ".%scache" % app_name) cache = {} try: @@ -192,18 +194,18 @@ def load_cache(self): return cache - def save_cache(self): - path = os.path.join(self.output, '.clarcache') + def save_cache(self, app_name): + path = os.path.join(self.output, ".%scache" % app_name) with open(path, 'wb') as cache: pickle.dump(self.modules, cache) - def load(self, force = False): + def load(self, app_name, prefix, force = False): module_data = self.find_modules() - self.modules = {} if force else self.load_cache() + self.modules = {} if force else self.load_cache(app_name) for path, name in module_data: if name not in self.modules: - self.modules[name] = Module(name) + self.modules[name] = Module(name, app_name, prefix) if not self.modules[name].refresh(path): del self.modules[name] @@ -222,15 +224,15 @@ def suite_count(self): def callback_count(self): return sum(len(module.callbacks) for module in self.modules.values()) - def write(self): + def write(self, name): if not os.path.exists(self.output): os.makedirs(self.output) - wrote_suite = self.write_suite() - wrote_header = self.write_header() + wrote_suite = self.write_suite(name) + wrote_header = self.write_header(name) if wrote_suite or wrote_header: - self.save_cache() + self.save_cache(name) return True return False @@ -257,8 +259,8 @@ def write_output(self, fn, data): return True - def write_suite(self): - suite_fn = os.path.join(self.output, 'clar.suite') + def write_suite(self, name): + suite_fn = os.path.join(self.output, '%s.suite' % name) with io.StringIO() as suite_file: modules = sorted(self.modules.values(), key=lambda module: module.name) @@ -271,25 +273,26 @@ def write_suite(self): t = Module.CallbacksTemplate(module) suite_file.write(t.render()) - suites = "static struct clar_suite _clar_suites[] = {" + ','.join( + suites = "static struct %s_suite _%s_suites[] = {" % (name, name) + suites += ','.join( Module.InfoTemplate(module).render() for module in modules ) + "\n};\n" suite_file.write(suites) - suite_file.write(u"static const size_t _clar_suite_count = %d;\n" % self.suite_count()) - suite_file.write(u"static const size_t _clar_callback_count = %d;\n" % self.callback_count()) + suite_file.write(u"static const size_t _%s_suite_count = %d;\n" % (name, self.suite_count())) + suite_file.write(u"static const size_t _%s_callback_count = %d;\n" % (name, self.callback_count())) return self.write_output(suite_fn, suite_file.getvalue()) return False - def write_header(self): - header_fn = os.path.join(self.output, 'clar_suite.h') + def write_header(self, name): + header_fn = os.path.join(self.output, '%s_suite.h' % name) with io.StringIO() as header_file: - header_file.write(u"#ifndef _____clar_suite_h_____\n") - header_file.write(u"#define _____clar_suite_h_____\n") + header_file.write(u"#ifndef _____%s_suite_h_____\n" % name) + header_file.write(u"#define _____%s_suite_h_____\n" % name) modules = sorted(self.modules.values(), key=lambda module: module.name) @@ -310,6 +313,8 @@ def write_header(self): parser.add_option('-f', '--force', action="store_true", dest='force', default=False) parser.add_option('-x', '--exclude', dest='excluded', action='append', default=[]) parser.add_option('-o', '--output', dest='output') + parser.add_option('-n', '--name', dest='name', default='clar') + parser.add_option('-p', '--prefix', dest='prefix', default='test') options, args = parser.parse_args() if len(args) > 1: @@ -323,7 +328,8 @@ def write_header(self): output = options.output or path suite = TestSuite(path, output) - suite.load(options.force) + suite.load(options.name, options.prefix, options.force) suite.disable(options.excluded) - if suite.write(): - print("Written `clar.suite`, `clar_suite.h` (%d tests in %d suites)" % (suite.callback_count(), suite.suite_count())) + + if suite.write(options.name): + print("Written `%s.suite`, `%s_suite.h` (%d tests in %d suites)" % (options.name, options.name, suite.callback_count(), suite.suite_count())) From b0a2e14018479c56f546030f3954156c4466caf3 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 17 Jan 2025 13:15:07 +0000 Subject: [PATCH 02/10] tests have optional metadata (eg descriptions) Tests can now have optional metadata, provided as comments in the test definition. For example: ``` void test_spline__reticulation(void) /*[clar]: description="ensure that splines are reticulated" */ { ... } ``` This description is preserved and produced as part of the summary XML. --- clar.c | 15 +++++++++++---- clar/summary.h | 19 +++++++++++++++---- generate.py | 47 ++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 66 insertions(+), 15 deletions(-) diff --git a/clar.c b/clar.c index e959a5a..b1d0bad 100644 --- a/clar.c +++ b/clar.c @@ -130,9 +130,10 @@ struct clar_explicit { }; struct clar_report { + const char *suite; const char *test; + const char *description; int test_number; - const char *suite; enum cl_test_status status; time_t start; @@ -152,8 +153,9 @@ struct clar_summary { static struct { enum cl_test_status test_status; - const char *active_test; const char *active_suite; + const char *active_test; + const char *active_description; int total_skipped; int total_errors; @@ -193,6 +195,7 @@ static struct { struct clar_func { const char *name; + const char *description; void (*ptr)(void); }; @@ -384,6 +387,7 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) _clar.active_suite = suite->name; _clar.active_test = NULL; + _clar.active_description = NULL; CL_TRACE(CL_TRACE__SUITE_BEGIN); if (filter) { @@ -412,11 +416,13 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) continue; _clar.active_test = test[i].name; + _clar.active_description = test[i].description; if ((report = calloc(1, sizeof(*report))) == NULL) clar_abort("Failed to allocate report.\n"); report->suite = _clar.active_suite; report->test = _clar.active_test; + report->description = _clar.active_description; report->test_number = _clar.tests_ran; report->status = CL_TEST_NOTRUN; @@ -435,6 +441,7 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) } _clar.active_test = NULL; + _clar.active_description = NULL; CL_TRACE(CL_TRACE__SUITE_END); } @@ -792,13 +799,13 @@ void clar__assert( const char *function, size_t line, const char *error_msg, - const char *description, + const char *error_description, int should_abort) { if (condition) return; - clar__fail(file, function, line, error_msg, description, should_abort); + clar__fail(file, function, line, error_msg, error_description, should_abort); } void clar__assert_equal( diff --git a/clar/summary.h b/clar/summary.h index 7b85f16..d4dea1e 100644 --- a/clar/summary.h +++ b/clar/summary.h @@ -42,11 +42,22 @@ static int clar_summary_testsuite(struct clar_summary *summary, } static int clar_summary_testcase(struct clar_summary *summary, - const char *name, const char *classname, double elapsed) + const struct clar_report *report) { - return fprintf(summary->fp, + if (fprintf(summary->fp, "\t\t\n", - name, classname, elapsed); + report->test, report->suite, report->elapsed) < 0) + return -1; + + if (report->description) { + fprintf(summary->fp, "\t\t\t\n"); + fprintf(summary->fp, "\t\t\t\t\n"); + fprintf(summary->fp, "\t\t\t\t\t%s\n", report->description); + fprintf(summary->fp, "\t\t\t\t\n"); + fprintf(summary->fp, "\t\t\t\n"); + } + + return 0; } static int clar_summary_failure(struct clar_summary *summary, @@ -100,7 +111,7 @@ int clar_summary_shutdown(struct clar_summary *summary) last_suite = report->suite; - clar_summary_testcase(summary, report->test, report->suite, report->elapsed); + clar_summary_testcase(summary, report); while (error != NULL) { if (clar_summary_failure(summary, "assert", diff --git a/generate.py b/generate.py index 7f40979..1cb6057 100755 --- a/generate.py +++ b/generate.py @@ -17,8 +17,12 @@ def __init__(self, module): def _render_callback(self, cb): if not cb: - return ' { NULL, NULL }' - return ' { "%s", &%s }' % (cb['short_name'], cb['symbol']) + return ' { NULL, NULL, NULL }' + + return ' { "%s", %s, &%s }' % \ + (cb['short_name'], \ + '"' + cb['description'] + '"' if cb['description'] != None else "NULL", \ + cb['symbol']) class DeclarationTemplate(Template): def render(self): @@ -87,7 +91,7 @@ def clean_name(self): def _skip_comments(self, text): SKIP_COMMENTS_REGEX = re.compile( - r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', + r'//.*?$|/\*(?!\s*\[clar\]:).*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', re.DOTALL | re.MULTILINE) def _replacer(match): @@ -97,20 +101,49 @@ def _replacer(match): return re.sub(SKIP_COMMENTS_REGEX, _replacer, text) def parse(self, contents): - TEST_FUNC_REGEX = r"^(void\s+(test_%s__(\w+))\s*\(\s*void\s*\))\s*\{" + TEST_FUNC_REGEX = r"^(void\s+(%s_%s__(\w+))\s*\(\s*void\s*\))(?:\s*/\*\s*\[clar\]:\s*(.*?)\s*\*/)?\s*\{" contents = self._skip_comments(contents) - regex = re.compile(TEST_FUNC_REGEX % self.name, re.MULTILINE) + regex = re.compile(TEST_FUNC_REGEX % (self.prefix, self.name), re.MULTILINE) self.callbacks = [] self.initializers = [] self.cleanup = None - for (declaration, symbol, short_name) in regex.findall(contents): + for (declaration, symbol, short_name, options) in regex.findall(contents): + description = None + + while options != '': + match = re.search(r'^([a-zA-Z0-9]+)=(\"[^"]*\"|[a-zA-Z0-9_\-]+|\d+)(?:,\s*|\Z)(.*)', options) + + if match == None: + print("Invalid options: '%s' for '%s'" % (options, symbol)) + sys.exit(1) + + key = match.group(1) + value = match.group(2) + options = match.group(3) + + match = re.search(r'^\"(.*)\"$', value) + if match != None: + value = match.group(1) + + match = re.search(r'([^a-zA-Z0-9 _\-,\.])', value) + if match != None: + print("Invalid character '%s' in %s for '%s'" % (match.group(1), key, symbol)) + sys.exit(1) + + if key == "description": + description = value + else: + print("Invalid option: '%s' for '%s'" % (key, symbol)) + sys.exit(1) + data = { "short_name" : short_name, "declaration" : declaration, - "symbol" : symbol + "symbol" : symbol, + "description" : description } if short_name.startswith('initialize'): From 486f0b335b28d93e8698a66458b51d144c868769 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 17 Jan 2025 23:57:27 +0000 Subject: [PATCH 03/10] use monotonic performance counter for elapsed time Move the elapsed time calculation to `counter.h`, and use high-resolution monotonic performance counters on all platforms. --- clar.c | 44 +++---------- clar/counter.h | 167 +++++++++++++++++++++++++++++++++++++++++++++++++ clar/summary.h | 2 +- 3 files changed, 177 insertions(+), 36 deletions(-) create mode 100644 clar/counter.h diff --git a/clar.c b/clar.c index b1d0bad..ec310dc 100644 --- a/clar.c +++ b/clar.c @@ -208,7 +208,7 @@ struct clar_suite { int enabled; }; -/* From clar_print_*.c */ +/* From print.h */ static void clar_print_init(int test_count, int suite_count); static void clar_print_shutdown(int test_count, int suite_count, int error_count); static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error); @@ -217,7 +217,7 @@ static void clar_print_onsuite(const char *suite_name, int suite_index); static void clar_print_onabortv(const char *msg, va_list argp); static void clar_print_onabort(const char *msg, ...); -/* From clar_sandbox.c */ +/* From sandbox.c */ static void clar_tempdir_init(void); static void clar_tempdir_shutdown(void); static int clar_sandbox_create(const char *suite_name, const char *test_name); @@ -227,6 +227,8 @@ static int clar_sandbox_cleanup(void); static struct clar_summary *clar_summary_init(const char *filename); static int clar_summary_shutdown(struct clar_summary *fp); +#include "clar/counter.h" + /* Load the declarations for the test suite */ #include "clar.suite" @@ -283,35 +285,6 @@ clar_report_all(void) } } -#ifdef WIN32 -# define clar_time DWORD - -static void clar_time_now(clar_time *out) -{ - *out = GetTickCount(); -} - -static double clar_time_diff(clar_time *start, clar_time *end) -{ - return ((double)*end - (double)*start) / 1000; -} -#else -# include - -# define clar_time struct timeval - -static void clar_time_now(clar_time *out) -{ - gettimeofday(out, NULL); -} - -static double clar_time_diff(clar_time *start, clar_time *end) -{ - return ((double)end->tv_sec + (double)end->tv_usec / 1.0E6) - - ((double)start->tv_sec + (double)start->tv_usec / 1.0E6); -} -#endif - static void clar_run_test( const struct clar_suite *suite, @@ -319,16 +292,17 @@ clar_run_test( const struct clar_func *initialize, const struct clar_func *cleanup) { - clar_time start, end; + struct clar_counter start, end; _clar.trampoline_enabled = 1; + _clar.last_report->start = time(NULL); CL_TRACE(CL_TRACE__TEST__BEGIN); clar_sandbox_create(suite->name, test->name); _clar.last_report->start = time(NULL); - clar_time_now(&start); + clar_counter_now(&start); if (setjmp(_clar.trampoline) == 0) { if (initialize->ptr != NULL) @@ -339,14 +313,14 @@ clar_run_test( CL_TRACE(CL_TRACE__TEST__RUN_END); } - clar_time_now(&end); + clar_counter_now(&end); _clar.trampoline_enabled = 0; if (_clar.last_report->status == CL_TEST_NOTRUN) _clar.last_report->status = CL_TEST_OK; - _clar.last_report->elapsed = clar_time_diff(&start, &end); + _clar.last_report->elapsed = clar_counter_diff(&start, &end); if (_clar.local_cleanup != NULL) _clar.local_cleanup(_clar.local_cleanup_payload); diff --git a/clar/counter.h b/clar/counter.h new file mode 100644 index 0000000..89b43d8 --- /dev/null +++ b/clar/counter.h @@ -0,0 +1,167 @@ +#define CLAR_COUNTER_TV_DIFF(out_sec, out_usec, start_sec, start_usec, end_sec, end_usec) \ + if (start_usec > end_usec) { \ + out_sec = (end_sec - 1) - start_sec; \ + out_usec = (end_usec + 1000000) - start_usec; \ + } else { \ + out_sec = end_sec - start_sec; \ + out_usec = end_usec - start_usec; \ + } + +#ifdef _WIN32 + +struct clar_counter { + LARGE_INTEGER value; +}; + +void clar_counter_now(struct clar_counter *out) +{ + QueryPerformanceCounter(&out->value); +} + +double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + LARGE_INTEGER freq; + + QueryPerformanceFrequency(&freq); + return (double)(end->value.QuadPart - start->value.QuadPart)/(double)freq.QuadPart; +} + +#elif __APPLE__ + +#include +#include + +static double clar_counter_scaling_factor = -1; + +struct clar_counter { + union { + uint64_t absolute_time; + struct timeval tv; + } val; +}; + +static void clar_counter_now(struct clar_counter *out) +{ + if (clar_counter_scaling_factor == 0) { + mach_timebase_info_data_t info; + + clar_counter_scaling_factor = + mach_timebase_info(&info) == KERN_SUCCESS ? + ((double)info.numer / (double)info.denom) / 1.0E6 : + -1; + } + + /* mach_timebase_info failed; fall back to gettimeofday */ + if (clar_counter_scaling_factor < 0) + gettimeofday(&out->val.tv, NULL); + else + out->val.absolute_time = mach_absolute_time(); +} + +static double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + if (clar_counter_scaling_factor < 0) { + time_t sec; + suseconds_t usec; + + CLAR_COUNTER_TV_DIFF(sec, usec, + start->val.tv.tv_sec, start->val.tv.tv_usec, + end->val.tv.tv_sec, end->val.tv.tv_usec); + + return (double)sec + ((double)usec / 1000000.0); + } else { + return (double)(end->val.absolute_time - start->val.absolute_time) * + clar_counter_scaling_factor; + } +} + +#elif defined(__amigaos4__) + +#include + +struct clar_counter { + struct TimeVal tv; +} + +static void clar_counter_now(struct clar_counter *out) +{ + ITimer->GetUpTime(&out->tv); +} + +static double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + uint32_t sec, usec; + + CLAR_COUNTER_TV_DIFF(sec, usec, + start->tv.Seconds, start->tv.Microseconds, + end->tv.Seconds, end->tv.Microseconds); + + return (double)sec + ((double)usec / 1000000.0); +} + +#else + +#include + +struct clar_counter { + int type; + union { +#ifdef CLOCK_MONOTONIC + struct timespec tp; +#endif + struct timeval tv; + } val; +}; + +static void clar_counter_now(struct clar_counter *out) +{ +#ifdef CLOCK_MONOTONIC + if (clock_gettime(CLOCK_MONOTONIC, &out->val.tp) == 0) { + out->type = 0; + return; + } +#endif + + /* Fall back to using gettimeofday */ + out->type = 1; + gettimeofday(&out->val.tv, NULL); +} + +static double clar_counter_diff( + struct clar_counter *start, + struct clar_counter *end) +{ + time_t sec; + suseconds_t usec; + +#ifdef CLOCK_MONOTONIC + if (start->type == 0) { + time_t sec; + long nsec; + + if (start->val.tp.tv_sec > end->val.tp.tv_sec) { + sec = (end->val.tp.tv_sec - 1) - start->val.tp.tv_sec; + nsec = (end->val.tp.tv_nsec + 1000000000) - start->val.tp.tv_nsec; + } else { + sec = end->val.tp.tv_sec - start->val.tp.tv_sec; + nsec = end->val.tp.tv_nsec - start->val.tp.tv_nsec; + } + + return (double)sec + ((double)nsec / 1000000000.0); + } +#endif + + CLAR_COUNTER_TV_DIFF(sec, usec, + start->val.tv.tv_sec, start->val.tv.tv_usec, + end->val.tv.tv_sec, end->val.tv.tv_usec); + + return (double)sec + ((double)usec / 1000000.0); +} + +#endif diff --git a/clar/summary.h b/clar/summary.h index d4dea1e..648cdc9 100644 --- a/clar/summary.h +++ b/clar/summary.h @@ -45,7 +45,7 @@ static int clar_summary_testcase(struct clar_summary *summary, const struct clar_report *report) { if (fprintf(summary->fp, - "\t\t\n", + "\t\t\n", report->test, report->suite, report->elapsed) < 0) return -1; From c829a9247a3f528f009fa4c1aa46b2c6ded24d3c Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 18 Jan 2025 12:16:25 +0000 Subject: [PATCH 04/10] add test_start print callback Refactor the `ontest` callback (which is implicitly test _finished_) into a test started and test finished callback. This allows printers to show the test name (at start) and its conclusion in two steps, which is advantageous for users to see the current test during long-running test executions. In addition, rename `onsuite` to `suite_start` for consistency. --- clar.c | 11 +++++----- clar/print.h | 57 +++++++++++++++++++++++++++++++++++----------------- 2 files changed, 45 insertions(+), 23 deletions(-) diff --git a/clar.c b/clar.c index ec310dc..e77428f 100644 --- a/clar.c +++ b/clar.c @@ -212,8 +212,9 @@ struct clar_suite { static void clar_print_init(int test_count, int suite_count); static void clar_print_shutdown(int test_count, int suite_count, int error_count); static void clar_print_error(int num, const struct clar_report *report, const struct clar_error *error); -static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status failed); -static void clar_print_onsuite(const char *suite_name, int suite_index); +static void clar_print_suite_start(const char *suite_name, int suite_index); +static void clar_print_test_start(const char *suite_name, const char *test_name, int test_number); +static void clar_print_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report); static void clar_print_onabortv(const char *msg, va_list argp); static void clar_print_onabort(const char *msg, ...); @@ -301,7 +302,7 @@ clar_run_test( clar_sandbox_create(suite->name, test->name); - _clar.last_report->start = time(NULL); + clar_print_test_start(suite->name, test->name, _clar.tests_ran); clar_counter_now(&start); if (setjmp(_clar.trampoline) == 0) { @@ -340,7 +341,7 @@ clar_run_test( _clar.local_cleanup = NULL; _clar.local_cleanup_payload = NULL; - clar_print_ontest(suite->name, test->name, _clar.tests_ran, _clar.last_report->status); + clar_print_test_finish(suite->name, test->name, _clar.tests_ran, _clar.last_report); } static void @@ -357,7 +358,7 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) if (_clar.exit_on_error && _clar.total_errors) return; - clar_print_onsuite(suite->name, ++_clar.suites_ran); + clar_print_suite_start(suite->name, ++_clar.suites_ran); _clar.active_suite = suite->name; _clar.active_test = NULL; diff --git a/clar/print.h b/clar/print.h index 6a2321b..5783e29 100644 --- a/clar/print.h +++ b/clar/print.h @@ -57,9 +57,8 @@ static void clar_print_clap_error(int num, const struct clar_report *report, con fflush(stdout); } -static void clar_print_clap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_clap_test_start(const char *suite_name, const char *test_name, int test_number) { - (void)test_name; (void)test_number; if (_clar.verbosity < 0) @@ -67,15 +66,18 @@ static void clar_print_clap_ontest(const char *suite_name, const char *test_name if (_clar.verbosity > 1) { printf("%s::%s: ", suite_name, test_name); + fflush(stdout); + } +} - switch (status) { - case CL_TEST_OK: printf("ok\n"); break; - case CL_TEST_FAILURE: printf("fail\n"); break; - case CL_TEST_SKIP: printf("skipped\n"); break; - case CL_TEST_NOTRUN: printf("notrun\n"); break; - } - } else { - switch (status) { +static void clar_print_clap_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) +{ + (void)suite_name; + (void)test_name; + (void)test_number; + + if (_clar.verbosity == 0) { + switch (report->status) { case CL_TEST_OK: printf("."); break; case CL_TEST_FAILURE: printf("F"); break; case CL_TEST_SKIP: printf("S"); break; @@ -83,10 +85,17 @@ static void clar_print_clap_ontest(const char *suite_name, const char *test_name } fflush(stdout); + } else if (_clar.verbosity > 1) { + switch (report->status) { + case CL_TEST_OK: printf("ok\n"); break; + case CL_TEST_FAILURE: printf("fail\n"); break; + case CL_TEST_SKIP: printf("skipped\n"); break; + case CL_TEST_NOTRUN: printf("notrun\n"); break; + } } } -static void clar_print_clap_onsuite(const char *suite_name, int suite_index) +static void clar_print_clap_suite_start(const char *suite_name, int suite_index) { if (_clar.verbosity < 0) return; @@ -138,14 +147,21 @@ static void print_escaped(const char *str) printf("%s", str); } -static void clar_print_tap_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_tap_test_start(const char *suite_name, const char *test_name, int test_number) +{ + (void)suite_name; + (void)test_name; + (void)test_number; +} + +static void clar_print_tap_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) { const struct clar_error *error = _clar.last_report->errors; (void)test_name; (void)test_number; - switch(status) { + switch(report->status) { case CL_TEST_OK: printf("ok %d - %s::%s\n", test_number, suite_name, test_name); break; @@ -177,7 +193,7 @@ static void clar_print_tap_ontest(const char *suite_name, const char *test_name, fflush(stdout); } -static void clar_print_tap_onsuite(const char *suite_name, int suite_index) +static void clar_print_tap_suite_start(const char *suite_name, int suite_index) { if (_clar.verbosity < 0) return; @@ -221,14 +237,19 @@ static void clar_print_error(int num, const struct clar_report *report, const st PRINT(error, num, report, error); } -static void clar_print_ontest(const char *suite_name, const char *test_name, int test_number, enum cl_test_status status) +static void clar_print_test_start(const char *suite_name, const char *test_name, int test_number) +{ + PRINT(test_start, suite_name, test_name, test_number); +} + +static void clar_print_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) { - PRINT(ontest, suite_name, test_name, test_number, status); + PRINT(test_finish, suite_name, test_name, test_number, report); } -static void clar_print_onsuite(const char *suite_name, int suite_index) +static void clar_print_suite_start(const char *suite_name, int suite_index) { - PRINT(onsuite, suite_name, suite_index); + PRINT(suite_start, suite_name, suite_index); } static void clar_print_onabortv(const char *msg, va_list argp) From cb3c6dff3413b8c7a7fb8f6aa05c62a6c270e17f Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 18 Jan 2025 12:50:42 +0000 Subject: [PATCH 05/10] multiple runs Allow tests to specify that they should have multiple runs. These runs all occur within a single initialization and cleanup phase, and is useful for repeatedly testing the same thing as quickly as possible. The time for each run is recorded, which may be useful for benchmarking that test run. --- CMakeLists.txt | 4 ++ clar.c | 83 ++++++++++++++++++++++++++++++++++---- clar/summary.h | 2 +- example/CMakeLists.txt | 2 +- generate.py | 14 +++++-- test/CMakeLists.txt | 2 +- test/suites/CMakeLists.txt | 2 +- 7 files changed, 94 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 125db05..d1f28b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,6 +27,10 @@ set_target_properties(clar PROPERTIES C_EXTENSIONS OFF ) +if(NOT WIN32) + set(CLAR_LIBRARIES m) +endif() + if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) include(CTest) if(BUILD_TESTING) diff --git a/clar.c b/clar.c index e77428f..7ff9f50 100644 --- a/clar.c +++ b/clar.c @@ -135,9 +135,17 @@ struct clar_report { const char *description; int test_number; + int runs; + enum cl_test_status status; time_t start; - double elapsed; + + double *times; + double time_min; + double time_max; + double time_mean; + double time_stddev; + double time_total; struct clar_error *errors; struct clar_error *last_error; @@ -196,6 +204,7 @@ static struct { struct clar_func { const char *name; const char *description; + int runs; void (*ptr)(void); }; @@ -286,6 +295,43 @@ clar_report_all(void) } } +static void +compute_times(void) +{ + double total_squares = 0; + int i; + + _clar.last_report->time_min = _clar.last_report->times[0]; + _clar.last_report->time_max = _clar.last_report->times[0]; + _clar.last_report->time_total = _clar.last_report->times[0]; + + for (i = 1; i < _clar.last_report->runs; i++) { + if (_clar.last_report->times[i] < _clar.last_report->time_min) + _clar.last_report->time_min = _clar.last_report->times[i]; + + if (_clar.last_report->times[i] > _clar.last_report->time_max) + _clar.last_report->time_max = _clar.last_report->times[i]; + + _clar.last_report->time_total += _clar.last_report->times[i]; + } + + if (_clar.last_report->runs <= 1) { + _clar.last_report->time_stddev = 0; + } else { + _clar.last_report->time_mean = _clar.last_report->time_total / _clar.last_report->runs; + + for (i = 0; i < _clar.last_report->runs; i++) { + double dev = (_clar.last_report->times[i] > _clar.last_report->time_mean) ? + _clar.last_report->times[i] - _clar.last_report->time_mean : + _clar.last_report->time_mean - _clar.last_report->times[i]; + + total_squares += (dev * dev); + } + + _clar.last_report->time_stddev = sqrt(total_squares / _clar.last_report->runs); + } +} + static void clar_run_test( const struct clar_suite *suite, @@ -293,9 +339,17 @@ clar_run_test( const struct clar_func *initialize, const struct clar_func *cleanup) { - struct clar_counter start, end; + int runs, i; + + runs = (test->runs > 0) ? test->runs : 1; + + _clar.last_report->times = (runs > 1) ? + calloc(runs, sizeof(double)) : + &_clar.last_report->time_mean; + + if (!_clar.last_report->times) + clar_abort("Failed to allocate report times.\n"); - _clar.trampoline_enabled = 1; _clar.last_report->start = time(NULL); CL_TRACE(CL_TRACE__TEST__BEGIN); @@ -303,25 +357,35 @@ clar_run_test( clar_sandbox_create(suite->name, test->name); clar_print_test_start(suite->name, test->name, _clar.tests_ran); - clar_counter_now(&start); + + _clar.trampoline_enabled = 1; if (setjmp(_clar.trampoline) == 0) { if (initialize->ptr != NULL) initialize->ptr(); CL_TRACE(CL_TRACE__TEST__RUN_BEGIN); - test->ptr(); + + for (i = 0; i < runs; i++) { + struct clar_counter start, end; + + clar_counter_now(&start); + test->ptr(); + clar_counter_now(&end); + + _clar.last_report->runs++; + _clar.last_report->times[i] = clar_counter_diff(&start, &end); + } + CL_TRACE(CL_TRACE__TEST__RUN_END); } - clar_counter_now(&end); - _clar.trampoline_enabled = 0; if (_clar.last_report->status == CL_TEST_NOTRUN) _clar.last_report->status = CL_TEST_OK; - _clar.last_report->elapsed = clar_counter_diff(&start, &end); + compute_times(); if (_clar.local_cleanup != NULL) _clar.local_cleanup(_clar.local_cleanup_payload); @@ -653,6 +717,9 @@ clar_test_shutdown(void) free(error); } + if (report->times != &report->time_mean) + free(report->times); + report_next = report->next; free(report); } diff --git a/clar/summary.h b/clar/summary.h index 648cdc9..55338e1 100644 --- a/clar/summary.h +++ b/clar/summary.h @@ -46,7 +46,7 @@ static int clar_summary_testcase(struct clar_summary *summary, { if (fprintf(summary->fp, "\t\t\n", - report->test, report->suite, report->elapsed) < 0) + report->test, report->suite, report->time_total) < 0) return -1; if (report->description) { diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index b72f187..4f81df3 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -25,4 +25,4 @@ target_include_directories(example PRIVATE "${CMAKE_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" ) -target_link_libraries(example clar) +target_link_libraries(example clar ${CLAR_LIBRARIES}) diff --git a/generate.py b/generate.py index 1cb6057..67bf7bc 100755 --- a/generate.py +++ b/generate.py @@ -17,11 +17,12 @@ def __init__(self, module): def _render_callback(self, cb): if not cb: - return ' { NULL, NULL, NULL }' + return ' { NULL, NULL, 0, NULL }' - return ' { "%s", %s, &%s }' % \ + return ' { "%s", %s, %d, &%s }' % \ (cb['short_name'], \ '"' + cb['description'] + '"' if cb['description'] != None else "NULL", \ + cb['runs'], \ cb['symbol']) class DeclarationTemplate(Template): @@ -111,6 +112,7 @@ def parse(self, contents): self.cleanup = None for (declaration, symbol, short_name, options) in regex.findall(contents): + runs = 0 description = None while options != '': @@ -135,6 +137,11 @@ def parse(self, contents): if key == "description": description = value + elif key == "runs": + if not value.isnumeric(): + print("Invalid option: '%s' in runs for '%s'" % (option, symbol)) + sys.exit(1) + runs = int(value) else: print("Invalid option: '%s' for '%s'" % (key, symbol)) sys.exit(1) @@ -143,7 +150,8 @@ def parse(self, contents): "short_name" : short_name, "declaration" : declaration, "symbol" : symbol, - "description" : description + "description" : description, + "runs" : runs } if short_name.startswith('initialize'): diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f240166..88efde2 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -36,7 +36,7 @@ target_include_directories(selftest PRIVATE "${CMAKE_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" ) -target_link_libraries(selftest clar) +target_link_libraries(selftest clar ${CLAR_LIBRARIES}) add_test(NAME build_selftest COMMAND "${CMAKE_COMMAND}" --build "${CMAKE_BINARY_DIR}" --config "$" --target selftest diff --git a/test/suites/CMakeLists.txt b/test/suites/CMakeLists.txt index fa8ab94..5ea311f 100644 --- a/test/suites/CMakeLists.txt +++ b/test/suites/CMakeLists.txt @@ -44,7 +44,7 @@ foreach(suite IN LISTS suites) "${CMAKE_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/${suite}" ) - target_link_libraries(${suite}_suite clar) + target_link_libraries(${suite}_suite clar ${CLAR_LIBRARIES}) add_test(NAME build_${suite}_suite COMMAND "${CMAKE_COMMAND}" --build "${CMAKE_BINARY_DIR}" --config "$" --target selftest From 9f3d5d4cf1ee1e4f121eb62bf755dfa5e6cf9080 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 18 Jan 2025 17:08:36 +0000 Subject: [PATCH 06/10] add benchmark mode An application can provide _benchmarks_ instead of _tests_. Benchmarks can run multiple times, will calculate the times of each run, and some simple additional data (mean, min, max, etc). This information will be displayed and will optionally be emitted in the summary output. Test hosts can indicate that they're benchmarks (not tests) by setting the mode before parsing the arguments. This will switch the output and summary format types. --- clar.c | 16 ++++ clar.h | 12 +++ clar/print.h | 111 +++++++++++++++++++++++ clar/summary.h | 234 +++++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 336 insertions(+), 37 deletions(-) diff --git a/clar.c b/clar.c index 7ff9f50..eeb901a 100644 --- a/clar.c +++ b/clar.c @@ -159,6 +159,7 @@ struct clar_summary { }; static struct { + enum cl_test_mode test_mode; enum cl_test_status test_status; const char *active_suite; @@ -172,6 +173,7 @@ static struct { int suites_ran; enum cl_output_format output_format; + enum cl_summary_format summary_format; int exit_on_error; int verbosity; @@ -646,6 +648,14 @@ clar_test_init(int argc, char **argv) { const char *summary_env; + if (_clar.test_mode == CL_TEST_BENCHMARK) { + _clar.output_format = CL_OUTPUT_TIMING; + _clar.summary_format = CL_SUMMARY_JSON; + } else { + _clar.output_format = CL_OUTPUT_CLAP; + _clar.summary_format = CL_SUMMARY_JUNIT; + } + if (argc > 1) clar_parse_args(argc, argv); @@ -668,6 +678,12 @@ clar_test_init(int argc, char **argv) clar_tempdir_init(); } +void +clar_test_set_mode(enum cl_test_mode mode) +{ + _clar.test_mode = mode; +} + int clar_test_run(void) { diff --git a/clar.h b/clar.h index 9ea91d3..8ca25b7 100644 --- a/clar.h +++ b/clar.h @@ -31,6 +31,11 @@ # define CLAR_CURRENT_FUNC "func" #endif +enum cl_test_mode { + CL_TEST_STANDARD, + CL_TEST_BENCHMARK, +}; + enum cl_test_status { CL_TEST_OK, CL_TEST_FAILURE, @@ -41,10 +46,17 @@ enum cl_test_status { enum cl_output_format { CL_OUTPUT_CLAP, CL_OUTPUT_TAP, + CL_OUTPUT_TIMING, +}; + +enum cl_summary_format { + CL_SUMMARY_JUNIT, + CL_SUMMARY_JSON, }; /** Setup clar environment */ void clar_test_init(int argc, char *argv[]); +void clar_test_set_mode(enum cl_test_mode mode); int clar_test_run(void); void clar_test_shutdown(void); diff --git a/clar/print.h b/clar/print.h index 5783e29..ef57c22 100644 --- a/clar/print.h +++ b/clar/print.h @@ -207,6 +207,114 @@ static void clar_print_tap_onabort(const char *fmt, va_list arg) fflush(stdout); } +/* timings format: useful for benchmarks */ + +static void clar_print_timing_init(int test_count, int suite_count) +{ + (void)test_count; + (void)suite_count; + + printf("Started benchmarks (mean time ± stddev / min time … max time):\n\n"); +} + +static void clar_print_timing_shutdown(int test_count, int suite_count, int error_count) +{ + (void)test_count; + (void)suite_count; + (void)error_count; +} + +static void clar_print_timing_error(int num, const struct clar_report *report, const struct clar_error *error) +{ + (void)num; + (void)report; + (void)error; +} + +static void clar_print_timing_test_start(const char *suite_name, const char *test_name, int test_number) +{ + (void)test_number; + + printf("%s::%s: ", suite_name, test_name); + fflush(stdout); +} + +static void clar_print_timing_time(double t) +{ + static const char *units[] = { "sec", "ms", "μs", "ns" }; + static const int units_len = sizeof(units) / sizeof(units[0]); + int unit = 0, exponent = 0, digits; + + while (t < 1.0 && unit < units_len - 1) { + t *= 1000.0; + unit++; + } + + while (t > 0.0 && t < 1.0 && exponent < 10) { + t *= 10.0; + exponent++; + } + + digits = (t < 10.0) ? 3 : ((t < 100.0) ? 2 : 1); + + printf("%.*f", digits, t); + + if (exponent > 0) + printf("e-%d", exponent); + + printf(" %s", units[unit]); +} + +static void clar_print_timing_test_finish(const char *suite_name, const char *test_name, int test_number, const struct clar_report *report) +{ + const struct clar_error *error = _clar.last_report->errors; + + (void)suite_name; + (void)test_name; + (void)test_number; + + switch(report->status) { + case CL_TEST_OK: + clar_print_timing_time(report->time_mean); + + if (report->runs > 1) { + printf(" ± "); + clar_print_timing_time(report->time_stddev); + + printf(" / range: "); + clar_print_timing_time(report->time_min); + printf(" … "); + clar_print_timing_time(report->time_max); + printf(" (%d runs)", report->runs); + } + + printf("\n"); + break; + case CL_TEST_FAILURE: + printf("failed: %s\n", error->error_msg); + break; + case CL_TEST_SKIP: + case CL_TEST_NOTRUN: + printf("skipped\n"); + break; + } + + fflush(stdout); +} + +static void clar_print_timing_suite_start(const char *suite_name, int suite_index) +{ + if (_clar.verbosity == 1) + printf("\n%s", suite_name); + + (void)suite_index; +} + +static void clar_print_timing_onabort(const char *fmt, va_list arg) +{ + vfprintf(stderr, fmt, arg); +} + /* indirection between protocol output selection */ #define PRINT(FN, ...) do { \ @@ -217,6 +325,9 @@ static void clar_print_tap_onabort(const char *fmt, va_list arg) case CL_OUTPUT_TAP: \ clar_print_tap_##FN (__VA_ARGS__); \ break; \ + case CL_OUTPUT_TIMING: \ + clar_print_timing_##FN (__VA_ARGS__); \ + break; \ default: \ abort(); \ } \ diff --git a/clar/summary.h b/clar/summary.h index 55338e1..d7d2105 100644 --- a/clar/summary.h +++ b/clar/summary.h @@ -1,8 +1,24 @@ - #include #include -static int clar_summary_close_tag( +static int clar_summary_time_digits(double t) +{ + int digits = 3; + + if (t >= 100.0) + return 1; + else if (t >= 10.0) + return 2; + + while (t > 0.0 && t < 1.0 && digits < 10) { + t *= 10.0; + digits++; + } + + return digits; +} + +static int clar_summary_junit_close_tag( struct clar_summary *summary, const char *tag, int indent) { const char *indt; @@ -14,12 +30,12 @@ static int clar_summary_close_tag( return fprintf(summary->fp, "%s\n", indt, tag); } -static int clar_summary_testsuites(struct clar_summary *summary) +static int clar_summary_junit_testsuites(struct clar_summary *summary) { return fprintf(summary->fp, "\n"); } -static int clar_summary_testsuite(struct clar_summary *summary, +static int clar_summary_junit_testsuite(struct clar_summary *summary, int idn, const char *name, time_t timestamp, int test_count, int fail_count, int error_count) { @@ -41,26 +57,15 @@ static int clar_summary_testsuite(struct clar_summary *summary, idn, name, iso_dt, test_count, fail_count, error_count); } -static int clar_summary_testcase(struct clar_summary *summary, - const struct clar_report *report) +static int clar_summary_junit_testcase(struct clar_summary *summary, + const char *name, const char *classname, double elapsed) { - if (fprintf(summary->fp, - "\t\t\n", - report->test, report->suite, report->time_total) < 0) - return -1; - - if (report->description) { - fprintf(summary->fp, "\t\t\t\n"); - fprintf(summary->fp, "\t\t\t\t\n"); - fprintf(summary->fp, "\t\t\t\t\t%s\n", report->description); - fprintf(summary->fp, "\t\t\t\t\n"); - fprintf(summary->fp, "\t\t\t\n"); - } - - return 0; + return fprintf(summary->fp, + "\t\t\n", + name, classname, clar_summary_time_digits(elapsed), elapsed); } -static int clar_summary_failure(struct clar_summary *summary, +static int clar_summary_junit_failure(struct clar_summary *summary, const char *type, const char *message, const char *desc) { return fprintf(summary->fp, @@ -68,22 +73,26 @@ static int clar_summary_failure(struct clar_summary *summary, type, message, desc); } -static int clar_summary_skipped(struct clar_summary *summary) +static int clar_summary_junit_skipped(struct clar_summary *summary) { return fprintf(summary->fp, "\t\t\t\n"); } -struct clar_summary *clar_summary_init(const char *filename) +struct clar_summary *clar_summary_junit_init(const char *filename) { struct clar_summary *summary; FILE *fp; - if ((fp = fopen(filename, "w")) == NULL) - clar_abort("Failed to open the summary file '%s': %s.\n", - filename, strerror(errno)); + if ((fp = fopen(filename, "w")) == NULL) { + perror("fopen"); + return NULL; + } - if ((summary = malloc(sizeof(struct clar_summary))) == NULL) - clar_abort("Failed to allocate summary.\n"); + if ((summary = malloc(sizeof(struct clar_summary))) == NULL) { + perror("malloc"); + fclose(fp); + return NULL; + } summary->filename = filename; summary->fp = fp; @@ -91,12 +100,12 @@ struct clar_summary *clar_summary_init(const char *filename) return summary; } -int clar_summary_shutdown(struct clar_summary *summary) +int clar_summary_junit_shutdown(struct clar_summary *summary) { struct clar_report *report; const char *last_suite = NULL; - if (clar_summary_testsuites(summary) < 0) + if (clar_summary_junit_testsuites(summary) < 0) goto on_error; report = _clar.reports; @@ -104,17 +113,17 @@ int clar_summary_shutdown(struct clar_summary *summary) struct clar_error *error = report->errors; if (last_suite == NULL || strcmp(last_suite, report->suite) != 0) { - if (clar_summary_testsuite(summary, 0, report->suite, + if (clar_summary_junit_testsuite(summary, 0, report->suite, report->start, _clar.tests_ran, _clar.total_errors, 0) < 0) goto on_error; } last_suite = report->suite; - clar_summary_testcase(summary, report); + clar_summary_junit_testcase(summary, report->test, report->suite, report->time_total); while (error != NULL) { - if (clar_summary_failure(summary, "assert", + if (clar_summary_junit_failure(summary, "assert", error->error_msg, error->description) < 0) goto on_error; @@ -122,20 +131,20 @@ int clar_summary_shutdown(struct clar_summary *summary) } if (report->status == CL_TEST_SKIP) - clar_summary_skipped(summary); + clar_summary_junit_skipped(summary); - if (clar_summary_close_tag(summary, "testcase", 2) < 0) + if (clar_summary_junit_close_tag(summary, "testcase", 2) < 0) goto on_error; report = report->next; if (!report || strcmp(last_suite, report->suite) != 0) { - if (clar_summary_close_tag(summary, "testsuite", 1) < 0) + if (clar_summary_junit_close_tag(summary, "testsuite", 1) < 0) goto on_error; } } - if (clar_summary_close_tag(summary, "testsuites", 0) < 0 || + if (clar_summary_junit_close_tag(summary, "testsuites", 0) < 0 || fclose(summary->fp) != 0) goto on_error; @@ -149,3 +158,154 @@ int clar_summary_shutdown(struct clar_summary *summary) free(summary); return -1; } + +struct clar_summary *clar_summary_json_init(const char *filename) +{ + struct clar_summary *summary; + FILE *fp; + + if ((fp = fopen(filename, "w")) == NULL) { + perror("fopen"); + return NULL; + } + + if ((summary = malloc(sizeof(struct clar_summary))) == NULL) { + perror("malloc"); + fclose(fp); + return NULL; + } + + summary->filename = filename; + summary->fp = fp; + + return summary; +} + +int clar_summary_json_shutdown(struct clar_summary *summary) +{ + struct clar_report *report; + int i; + + fprintf(summary->fp, "{\n"); + fprintf(summary->fp, " \"tests\": [\n"); + + report = _clar.reports; + while (report != NULL) { + struct clar_error *error = report->errors; + + if (report != _clar.reports) + fprintf(summary->fp, ",\n"); + + fprintf(summary->fp, " {\n"); + fprintf(summary->fp, " \"name\": \"%s::%s\",\n", report->suite, report->test); + + if (report->description) + fprintf(summary->fp, " \"description\": \"%s\",\n", report->description); + + fprintf(summary->fp, " \"results\": {\n"); + + fprintf(summary->fp, " \"status\": "); + if (report->status == CL_TEST_OK) + fprintf(summary->fp, "\"ok\",\n"); + else if (report->status == CL_TEST_FAILURE) + fprintf(summary->fp, "\"failed\",\n"); + else if (report->status == CL_TEST_SKIP) + fprintf(summary->fp, "\"skipped\"\n"); + else + clar_abort("unknown test status %d", report->status); + + if (report->status == CL_TEST_OK) { + fprintf(summary->fp, " \"mean\": %.*f,\n", + clar_summary_time_digits(report->time_mean), report->time_mean); + fprintf(summary->fp, " \"stddev\": %.*f,\n", + clar_summary_time_digits(report->time_stddev), report->time_stddev); + fprintf(summary->fp, " \"min\": %.*f,\n", + clar_summary_time_digits(report->time_min), report->time_min); + fprintf(summary->fp, " \"max\": %.*f,\n", + clar_summary_time_digits(report->time_max), report->time_max); + fprintf(summary->fp, " \"times\": [\n"); + + for (i = 0; i < report->runs; i++) { + if (i > 0) + fprintf(summary->fp, ",\n"); + + fprintf(summary->fp, " %.*f", + clar_summary_time_digits(report->times[i]), report->times[i]); + } + + fprintf(summary->fp, "\n ]\n"); + } + + if (report->status == CL_TEST_FAILURE) { + fprintf(summary->fp, " \"errors\": [\n"); + + while (error != NULL) { + if (error != report->errors) + fprintf(summary->fp, ",\n"); + + fprintf(summary->fp, " {\n"); + fprintf(summary->fp, " \"message\": \"%s\",\n", error->error_msg); + + if (error->description) + fprintf(summary->fp, " \"description\": \"%s\",\n", error->description); + + fprintf(summary->fp, " \"function\": \"%s\",\n", error->function); + fprintf(summary->fp, " \"file\": \"%s\",\n", error->file); + fprintf(summary->fp, " \"line\": %" PRIuMAX "\n", error->line_number); + fprintf(summary->fp, " }"); + + error = error->next; + } + + fprintf(summary->fp, "\n"); + fprintf(summary->fp, " ]\n"); + } + + fprintf(summary->fp, " }\n"); + fprintf(summary->fp, " }"); + + report = report->next; + } + + fprintf(summary->fp, "\n"); + fprintf(summary->fp, " ]\n"); + fprintf(summary->fp, "}\n"); + + if (fclose(summary->fp) != 0) + goto on_error; + + printf("written summary file to %s\n", summary->filename); + + free(summary); + return 0; + +on_error: + fclose(summary->fp); + free(summary); + return -1; +} + +/* indirection between protocol output selection */ + +#define SUMMARY(FN, ...) do { \ + switch (_clar.summary_format) { \ + case CL_SUMMARY_JUNIT: \ + return clar_summary_junit_##FN (__VA_ARGS__); \ + break; \ + case CL_SUMMARY_JSON: \ + return clar_summary_json_##FN (__VA_ARGS__); \ + break; \ + default: \ + abort(); \ + } \ + } while(0) + +struct clar_summary *clar_summary_init(const char *filename) +{ + SUMMARY(init, filename); +} + +int clar_summary_shutdown(struct clar_summary *summary) +{ + SUMMARY(shutdown, summary); +} From fae3b79b2df2ffc135250bd6f85eb3a143b92b30 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 18 Jan 2025 17:13:44 +0000 Subject: [PATCH 07/10] refactor error struct --- clar.c | 12 ++++++------ clar/print.h | 6 +++--- clar/summary.h | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/clar.c b/clar.c index eeb901a..8e476fb 100644 --- a/clar.c +++ b/clar.c @@ -113,11 +113,11 @@ fixture_path(const char *base, const char *fixture_name); #endif struct clar_error { - const char *file; + const char *message; + char *description; const char *function; + const char *file; uintmax_t line_number; - const char *error_msg; - char *description; struct clar_error *next; }; @@ -800,7 +800,7 @@ static void clar__failv( error->file = _clar.invoke_file ? _clar.invoke_file : file; error->function = _clar.invoke_func ? _clar.invoke_func : function; error->line_number = _clar.invoke_line ? _clar.invoke_line : line; - error->error_msg = error_msg; + error->message = error_msg; if (description != NULL) { va_list args_copy; @@ -856,14 +856,14 @@ void clar__assert( const char *file, const char *function, size_t line, - const char *error_msg, + const char *error_message, const char *error_description, int should_abort) { if (condition) return; - clar__fail(file, function, line, error_msg, error_description, should_abort); + clar__fail(file, function, line, error_message, error_description, should_abort); } void clar__assert_equal( diff --git a/clar/print.h b/clar/print.h index ef57c22..55aff5f 100644 --- a/clar/print.h +++ b/clar/print.h @@ -48,7 +48,7 @@ static void clar_print_clap_error(int num, const struct clar_report *report, con error->file, error->line_number); - clar_print_indented(error->error_msg, 2); + clar_print_indented(error->message, 2); if (error->description != NULL) clar_print_indented(error->description, 2); @@ -171,7 +171,7 @@ static void clar_print_tap_test_finish(const char *suite_name, const char *test_ if (_clar.verbosity >= 0) { printf(" ---\n"); printf(" reason: |\n"); - clar_print_indented(error->error_msg, 6); + clar_print_indented(error->message, 6); if (error->description) clar_print_indented(error->description, 6); @@ -291,7 +291,7 @@ static void clar_print_timing_test_finish(const char *suite_name, const char *te printf("\n"); break; case CL_TEST_FAILURE: - printf("failed: %s\n", error->error_msg); + printf("failed: %s\n", error->message); break; case CL_TEST_SKIP: case CL_TEST_NOTRUN: diff --git a/clar/summary.h b/clar/summary.h index d7d2105..ff1e6ff 100644 --- a/clar/summary.h +++ b/clar/summary.h @@ -124,7 +124,7 @@ int clar_summary_junit_shutdown(struct clar_summary *summary) while (error != NULL) { if (clar_summary_junit_failure(summary, "assert", - error->error_msg, error->description) < 0) + error->message, error->description) < 0) goto on_error; error = error->next; @@ -244,7 +244,7 @@ int clar_summary_json_shutdown(struct clar_summary *summary) fprintf(summary->fp, ",\n"); fprintf(summary->fp, " {\n"); - fprintf(summary->fp, " \"message\": \"%s\",\n", error->error_msg); + fprintf(summary->fp, " \"message\": \"%s\",\n", error->message); if (error->description) fprintf(summary->fp, " \"description\": \"%s\",\n", error->description); From b3c0df1528fe94a3f2bcabe8f7854f9dbe83785a Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 18 Jan 2025 17:42:52 +0000 Subject: [PATCH 08/10] in benchmark mode run iteratively In benchmark mode, when the number of runs was not explicitly specified in the test itself, run a reasonably number of iterations. We do this by measuring one run of the test, then using that data to determine how many iterations we should run to fit within 3 seconds. (With a minimum number of iterations to ensure that we get some data, and a maximum number to deal with poor precision for fast test runs.) The 3 second number, and 10 iteration minimum, were chosen by consulting the hyperfine defaults. --- clar.c | 39 ++++++++++++++++++++++++++------------- clar.h | 10 ++++++++++ 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/clar.c b/clar.c index 8e476fb..c06ab5d 100644 --- a/clar.c +++ b/clar.c @@ -100,6 +100,7 @@ typedef struct stat STAT_T; #endif +#define MIN(x, y) (((x) < (y)) ? (x) : (y)) #define MAX(x, y) (((x) > (y)) ? (x) : (y)) #include "clar.h" @@ -341,18 +342,10 @@ clar_run_test( const struct clar_func *initialize, const struct clar_func *cleanup) { - int runs, i; - - runs = (test->runs > 0) ? test->runs : 1; - - _clar.last_report->times = (runs > 1) ? - calloc(runs, sizeof(double)) : - &_clar.last_report->time_mean; - - if (!_clar.last_report->times) - clar_abort("Failed to allocate report times.\n"); + int runs = test->runs, i = 0; _clar.last_report->start = time(NULL); + _clar.last_report->times = &_clar.last_report->time_mean; CL_TRACE(CL_TRACE__TEST__BEGIN); @@ -368,16 +361,36 @@ clar_run_test( CL_TRACE(CL_TRACE__TEST__RUN_BEGIN); - for (i = 0; i < runs; i++) { + do { struct clar_counter start, end; + double elapsed; clar_counter_now(&start); test->ptr(); clar_counter_now(&end); + elapsed = clar_counter_diff(&start, &end); + + /* + * unless the number of runs was explicitly given + * in benchmark mode, use the first run as a sample + * to determine how many runs we should attempt + */ + if (_clar.test_mode == CL_TEST_BENCHMARK && !runs) { + runs = MAX(CLAR_BENCHMARK_RUN_MIN, (CLAR_BENCHMARK_RUN_TIME / elapsed)); + runs = MIN(CLAR_BENCHMARK_RUN_MAX, runs); + } + + if (i == 0 && runs > 1) { + _clar.last_report->times = calloc(runs, sizeof(double)); + + if (_clar.last_report->times == NULL) + clar_abort("Failed to allocate report times.\n"); + } + _clar.last_report->runs++; - _clar.last_report->times[i] = clar_counter_diff(&start, &end); - } + _clar.last_report->times[i] = elapsed; + } while(++i < runs); CL_TRACE(CL_TRACE__TEST__RUN_END); } diff --git a/clar.h b/clar.h index 8ca25b7..47609df 100644 --- a/clar.h +++ b/clar.h @@ -21,6 +21,16 @@ # define CLAR_MAX_PATH 4096 #endif +/* + * In benchmark mode, by default, clar will run the test repeatedly for + * approximately `CLAR_BENCHMARK_RUN_TIME` seconds, and at least + * `CLAR_BENCHMARK_RUN_MIN` iterations. + */ + +#define CLAR_BENCHMARK_RUN_TIME 3.0 +#define CLAR_BENCHMARK_RUN_MIN 10 +#define CLAR_BENCHMARK_RUN_MAX 30000000 + #ifndef CLAR_SELFTEST # define CLAR_CURRENT_FILE __FILE__ # define CLAR_CURRENT_LINE __LINE__ From 5d0ba302a484129a23c0a600c8e5a1edc22f02cd Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sun, 19 Jan 2025 10:53:32 +0000 Subject: [PATCH 09/10] introduce a reset function For multi-run tests (benchmarks), we introduce a `reset` function. By default, between each run of a test, the initialization will be called at startup, and the cleanup will be called at finish. A benchmark may wish to set up multi-run state at the beginning of the invocation (in initialization), and keep a steady state through all test runs. Users can now add a `reset` function so that initialization occurs only at the beginning of all runs. --- clar.c | 15 ++++++++++++++- generate.py | 8 ++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/clar.c b/clar.c index c06ab5d..4099eee 100644 --- a/clar.c +++ b/clar.c @@ -214,6 +214,7 @@ struct clar_func { struct clar_suite { const char *name; struct clar_func initialize; + struct clar_func reset; struct clar_func cleanup; const struct clar_func *tests; size_t test_count; @@ -340,6 +341,7 @@ clar_run_test( const struct clar_suite *suite, const struct clar_func *test, const struct clar_func *initialize, + const struct clar_func *reset, const struct clar_func *cleanup) { int runs = test->runs, i = 0; @@ -365,6 +367,17 @@ clar_run_test( struct clar_counter start, end; double elapsed; + if (i > 0 && reset->ptr != NULL) { + reset->ptr(); + } else if (i > 0) { + if (_clar.local_cleanup != NULL) + _clar.local_cleanup(_clar.local_cleanup_payload); + if (cleanup->ptr != NULL) + cleanup->ptr(); + if (initialize->ptr != NULL) + initialize->ptr(); + } + clar_counter_now(&start); test->ptr(); clar_counter_now(&end); @@ -488,7 +501,7 @@ clar_run_suite(const struct clar_suite *suite, const char *filter) _clar.last_report = report; - clar_run_test(suite, &test[i], &suite->initialize, &suite->cleanup); + clar_run_test(suite, &test[i], &suite->initialize, &suite->reset, &suite->cleanup); if (_clar.exit_on_error && _clar.total_errors) return; diff --git a/generate.py b/generate.py index 67bf7bc..2e4eaa0 100755 --- a/generate.py +++ b/generate.py @@ -32,6 +32,9 @@ def render(self): for initializer in self.module.initializers: out += "extern %s;\n" % initializer['declaration'] + if self.module.reset: + out += "extern %s;\n" % self.module.reset['declaration'] + if self.module.cleanup: out += "extern %s;\n" % self.module.cleanup['declaration'] @@ -63,12 +66,14 @@ def render(self): { "${clean_name}", ${initialize}, + ${reset}, ${cleanup}, ${cb_ptr}, ${cb_count}, ${enabled} }""" ).substitute( clean_name = name, initialize = self._render_callback(initializer), + reset = self._render_callback(self.module.reset), cleanup = self._render_callback(self.module.cleanup), cb_ptr = "_%s_cb_%s" % (self.module.app_name, self.module.name), cb_count = len(self.module.callbacks), @@ -109,6 +114,7 @@ def parse(self, contents): self.callbacks = [] self.initializers = [] + self.reset = None self.cleanup = None for (declaration, symbol, short_name, options) in regex.findall(contents): @@ -156,6 +162,8 @@ def parse(self, contents): if short_name.startswith('initialize'): self.initializers.append(data) + elif short_name == 'reset': + self.reset = data elif short_name == 'cleanup': self.cleanup = data else: From 977844ec7cbd501756dffb8bf9e2b0b1cec896d6 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sun, 19 Jan 2025 01:21:25 +0000 Subject: [PATCH 10/10] Update README with benchmark information --- README.md | 112 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 96 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 4159598..d2d681e 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,13 @@ Come out and Clar ================= -In Catalan, "clar" means clear, easy to perceive. Using clar will make it -easy to test and make clear the quality of your code. +Clar is a minimal C unit testing and benchmarking framework. It +provides a simple mechanism for writing tests and asserting +postconditions, while providing support for TAP and JUnit style +outputs. -> _Historical note_ -> -> Originally the clar project was named "clay" because the word "test" has its -> roots in the latin word *"testum"*, meaning "earthen pot", and *"testa"*, -> meaning "piece of burned clay"? -> -> This is because historically, testing implied melting metal in a pot to -> check its quality. Clay is what tests are made of. +In Catalan, "clar" means clear, easy to perceive. Using Clar will make it +easy to test and make clear the quality of your code. ## Quick Usage Overview @@ -19,9 +15,7 @@ Clar is a minimal C unit testing framework. It's been written to replace the old framework in [libgit2][libgit2], but it's both very versatile and straightforward to use. -Can you count to funk? - -- **Zero: Initialize test directory** +1. **Initialize test directory** ~~~~ sh $ mkdir tests @@ -29,7 +23,7 @@ Can you count to funk? $ cp $CLAR_ROOT/example/*.c tests ~~~~ -- **One: Write some tests** +2. **Write some tests** File: tests/adding.c: @@ -59,7 +53,7 @@ Can you count to funk? } ~~~~~ -- **Two: Build the test executable** +3. **Build the test executable** ~~~~ sh $ cd tests @@ -68,7 +62,7 @@ Can you count to funk? $ gcc -I. clar.c main.c adding.c -o testit ~~~~ -- **Funk: Funk it.** +4. **Run the tests** ~~~~ sh $ ./testit @@ -319,6 +313,92 @@ void test_example__a_test_with_auxiliary_methods(void) } ~~~~ +## Benchmarks + +The clar mixer (`generate.py`) and runner can also be used to support +simple benchmark capabilities. When running in benchmark mode, Clar +will run each test multiple times in succession, using a high-resolution +platform timer to measure the elapsed time of each run. + +By default, Clar will run each test repeatedly for 3 seconds (with +a minimum of 10 runs), but you can define the explicit number of +runs for each test in the definition. + +By default, Clar will run the initialization and cleanup functions +before _each_ test run. This allows for consistent setup and teardown +behavior, and predictability with existing test setups. However, you +can avoid this additional overhead by defining a _reset_ function. +This will be called between test runs instead of the cleanup and +re-initialization; in this case, initialization will occur only +before all test runs, and cleanup will be performed only when all +test runs are complete. + +To configure a benchmark application instead of a test application: + +1. **Set clar into benchmark mode in your main function** + + ~~~~ c + int main(int argc, char *argv[]) + { + clar_test_set_mode(CL_TEST_BENCHMARK); + clar_test_init(argc, argv); + res = clar_test_run(); + clar_test_shutdown(); + return res; + } + ~~~~ + +2. **Optionally, set up your initialization, cleanup, and reset + functions** + + ~~~~ c + void test_foo__initialize(void) + { + global_data = malloc(1024 * 1024 * 1024); + memset(global_data, 0, 1024 * 1024 * 1024); + } + + void test_foo__reset(void) + { + memset(global_data, 0, 1024 * 1024 * 1024); + } + + void test_foo__cleanup(void) + { + global_data = malloc(1024 * 1024 * 1024); + } + ~~~~ + +3. **Optionally, configure tests with a specific run number** + + ~~~~ c + /* Run this test 500 times */ + void test_foo__bar(void) + /* [clar]:runs=500 */ + { + bar(); + } + ~~~~ + +3. **Run the benchmarks** + + When running in benchmark mode, you'll see timings output; if you + write a summary file, it will be a JSON file that contains the + time information. + + ~~~~ sh + $ ./benchmarks -r/path/to/results.json + Started benchmarks (mean time ± stddev / min time … max time): + + foo::bar: 24.75 ms ± 1.214 ms / range: 24.41 ms … 38.06 ms (500 runs) + foo::baz: 24.67 ms ± 248.2 μs / range: 24.41 ms … 25.41 ms (478 runs) + foo::qux: 25.98 ms ± 333.0 μs / range: 25.64 ms … 26.82 ms (112 runs) + ~~~~ + +Note: you can change the prefix of the test function names from `test_` +to something of your choice by using the `--prefix=...` option for +the `generate.py` mixer script. + About Clar ==========