1/* tinytest.c -- Copyright 2009-2012 Nick Mathewson
2 *
3 * Redistribution and use in source and binary forms, with or without
4 * modification, are permitted provided that the following conditions
5 * are met:
6 * 1. Redistributions of source code must retain the above copyright
7 *    notice, this list of conditions and the following disclaimer.
8 * 2. Redistributions in binary form must reproduce the above copyright
9 *    notice, this list of conditions and the following disclaimer in the
10 *    documentation and/or other materials provided with the distribution.
11 * 3. The name of the author may not be used to endorse or promote products
12 *    derived from this software without specific prior written permission.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25#ifdef TINYTEST_LOCAL
26#include "tinytest_local.h"
27#endif
28
29#include <stdio.h>
30#include <stdlib.h>
31#include <string.h>
32#include <assert.h>
33
34#ifndef NO_FORKING
35
36#ifdef _WIN32
37#include <windows.h>
38#else
39#include <sys/types.h>
40#include <sys/wait.h>
41#include <unistd.h>
42#endif
43
44#if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
45#if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
46    __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
47/* Workaround for a stupid bug in OSX 10.6 */
48#define FORK_BREAKS_GCOV
49#include <vproc.h>
50#endif
51#endif
52
53#endif /* !NO_FORKING */
54
55#ifndef __GNUC__
56#define __attribute__(x)
57#endif
58
59#include "tinytest.h"
60#include "tinytest_macros.h"
61
62#define LONGEST_TEST_NAME 16384
63#define DEFAULT_TESTCASE_TIMEOUT 30U
64#define MAGIC_EXITCODE 42
65
66static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
67static int n_ok = 0; /**< Number of tests that have passed */
68static int n_bad = 0; /**< Number of tests that have failed. */
69static int n_skipped = 0; /**< Number of tests that have been skipped. */
70
71static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
72static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
73static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
74static unsigned int opt_timeout = DEFAULT_TESTCASE_TIMEOUT; /**< Timeout for every test (using alarm()) */
75const char *verbosity_flag = "";
76
77const struct testlist_alias_t *cfg_aliases=NULL;
78
79enum outcome { SKIP=2, OK=1, FAIL=0 };
80static enum outcome cur_test_outcome = 0;
81const char *cur_test_prefix = NULL; /**< prefix of the current test group */
82/** Name of the current test, if we haven't logged is yet. Used for --quiet */
83const char *cur_test_name = NULL;
84
85static void usage(struct testgroup_t *groups, int list_groups)
86	__attribute__((noreturn));
87static int process_test_option(struct testgroup_t *groups, const char *test);
88
89#ifdef _WIN32
90/* Copy of argv[0] for win32. */
91static char commandname[MAX_PATH+1];
92
93struct timeout_thread_args {
94	const testcase_fn *fn;
95	void *env;
96};
97
98static DWORD WINAPI
99timeout_thread_proc_(LPVOID arg)
100{
101	struct timeout_thread_args *args = arg;
102	(*(args->fn))(args->env);
103	ExitThread(cur_test_outcome == FAIL ? 1 : 0);
104}
105
106static enum outcome
107testcase_run_in_thread_(const struct testcase_t *testcase, void *env)
108{
109	/* We will never run testcase in a new thread when the
110	timeout is set to zero */
111	assert(opt_timeout);
112	DWORD ret, tid;
113	HANDLE handle;
114	struct timeout_thread_args args = {
115		&(testcase->fn),
116		env
117	};
118
119	handle =CreateThread(NULL, 0, timeout_thread_proc_,
120		(LPVOID)&args, 0, &tid);
121	ret = WaitForSingleObject(handle, opt_timeout * 1000U);
122	if (ret == WAIT_OBJECT_0) {
123		ret = 0;
124		if (!GetExitCodeThread(handle, &ret)) {
125			printf("GetExitCodeThread failed\n");
126			ret = 1;
127		}
128	} else if (ret == WAIT_TIMEOUT)	{
129		printf("timeout\n");
130	} else {
131		printf("Wait failed\n");
132	}
133	CloseHandle(handle);
134	if (ret == 0)
135		return OK;
136	else if (ret == MAGIC_EXITCODE)
137		return SKIP;
138	else
139		return FAIL;
140}
141#else
142static unsigned int testcase_set_timeout_(void)
143{
144	return alarm(opt_timeout);
145}
146
147static unsigned int testcase_reset_timeout_(void)
148{
149	return alarm(0);
150}
151#endif
152
153static enum outcome
154testcase_run_bare_(const struct testcase_t *testcase)
155{
156	void *env = NULL;
157	int outcome;
158	if (testcase->setup) {
159		env = testcase->setup->setup_fn(testcase);
160		if (!env)
161			return FAIL;
162		else if (env == (void*)TT_SKIP)
163			return SKIP;
164	}
165
166	cur_test_outcome = OK;
167	{
168		if (opt_timeout) {
169#ifdef _WIN32
170			cur_test_outcome = testcase_run_in_thread_(testcase, env);
171#else
172			testcase_set_timeout_();
173			testcase->fn(env);
174			testcase_reset_timeout_();
175#endif
176		} else {
177			testcase->fn(env);
178		}
179	}
180	outcome = cur_test_outcome;
181
182	if (testcase->setup) {
183		if (testcase->setup->cleanup_fn(testcase, env) == 0)
184			outcome = FAIL;
185	}
186
187	return outcome;
188}
189
190
191#ifndef NO_FORKING
192
193static enum outcome
194testcase_run_forked_(const struct testgroup_t *group,
195		     const struct testcase_t *testcase)
196{
197#ifdef _WIN32
198	/* Fork? On Win32?  How primitive!  We'll do what the smart kids do:
199	   we'll invoke our own exe (whose name we recall from the command
200	   line) with a command line that tells it to run just the test we
201	   want, and this time without forking.
202
203	   (No, threads aren't an option.  The whole point of forking is to
204	   share no state between tests.)
205	 */
206	int ok;
207	char buffer[LONGEST_TEST_NAME+256];
208	STARTUPINFOA si;
209	PROCESS_INFORMATION info;
210	DWORD ret;
211
212	if (!in_tinytest_main) {
213		printf("\nERROR.  On Windows, testcase_run_forked_ must be"
214		       " called from within tinytest_main.\n");
215		abort();
216	}
217	if (opt_verbosity>0)
218		printf("[forking] ");
219
220	snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s --timeout 0 %s%s",
221		 commandname, verbosity_flag, group->prefix, testcase->name);
222
223	memset(&si, 0, sizeof(si));
224	memset(&info, 0, sizeof(info));
225	si.cb = sizeof(si);
226
227	ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
228			   0, NULL, NULL, &si, &info);
229	if (!ok) {
230		printf("CreateProcess failed!\n");
231		return FAIL;
232	}
233	ret = WaitForSingleObject(info.hProcess,
234		(opt_timeout ? opt_timeout * 1000U : INFINITE));
235
236	if (ret == WAIT_OBJECT_0) {
237		GetExitCodeProcess(info.hProcess, &ret);
238	} else if (ret == WAIT_TIMEOUT) {
239		printf("timeout\n");
240	} else {
241		printf("Wait failed\n");
242	}
243	CloseHandle(info.hProcess);
244	CloseHandle(info.hThread);
245	if (ret == 0)
246		return OK;
247	else if (ret == MAGIC_EXITCODE)
248		return SKIP;
249	else
250		return FAIL;
251#else
252	int outcome_pipe[2];
253	pid_t pid;
254	(void)group;
255
256	if (pipe(outcome_pipe))
257		perror("opening pipe");
258
259	if (opt_verbosity>0)
260		printf("[forking] ");
261	pid = fork();
262#ifdef FORK_BREAKS_GCOV
263	vproc_transaction_begin(0);
264#endif
265	if (!pid) {
266		/* child. */
267		int test_r, write_r;
268		char b[1];
269		close(outcome_pipe[0]);
270		test_r = testcase_run_bare_(testcase);
271		assert(0<=(int)test_r && (int)test_r<=2);
272		b[0] = "NYS"[test_r];
273		write_r = (int)write(outcome_pipe[1], b, 1);
274		if (write_r != 1) {
275			perror("write outcome to pipe");
276			exit(1);
277		}
278		exit(0);
279		return FAIL; /* unreachable */
280	} else {
281		/* parent */
282		int status, r, exitcode;
283		char b[1];
284		/* Close this now, so that if the other side closes it,
285		 * our read fails. */
286		close(outcome_pipe[1]);
287		r = (int)read(outcome_pipe[0], b, 1);
288		if (r == 0) {
289			printf("[Lost connection!] ");
290			return FAIL;
291		} else if (r != 1) {
292			perror("read outcome from pipe");
293		}
294		waitpid(pid, &status, 0);
295		exitcode = WEXITSTATUS(status);
296		close(outcome_pipe[0]);
297		if (opt_verbosity>1)
298			printf("%s%s: exited with %i (%i)\n", group->prefix, testcase->name, exitcode, status);
299		if (exitcode != 0)
300		{
301			printf("[atexit failure!] ");
302			return FAIL;
303		}
304		return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
305	}
306#endif
307}
308
309#endif /* !NO_FORKING */
310
311int
312testcase_run_one(const struct testgroup_t *group,
313		 const struct testcase_t *testcase)
314{
315	enum outcome outcome;
316
317	if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) {
318		if (opt_verbosity>0)
319			printf("%s%s: %s\n",
320			   group->prefix, testcase->name,
321			   (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED");
322		++n_skipped;
323		return SKIP;
324	}
325
326	if (opt_verbosity>0 && !opt_forked) {
327		printf("%s%s: ", group->prefix, testcase->name);
328	} else {
329		if (opt_verbosity==0) printf(".");
330		cur_test_prefix = group->prefix;
331		cur_test_name = testcase->name;
332	}
333
334#ifndef NO_FORKING
335	if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
336		outcome = testcase_run_forked_(group, testcase);
337	} else {
338#else
339	{
340#endif
341		outcome = testcase_run_bare_(testcase);
342	}
343
344	if (outcome == OK) {
345		if (opt_verbosity>0 && !opt_forked)
346			puts(opt_verbosity==1?"OK":"");
347	} else if (outcome == SKIP) {
348		if (opt_verbosity>0 && !opt_forked)
349			puts("SKIPPED");
350	} else {
351		if (!opt_forked)
352			printf("\n  [%s FAILED]\n", testcase->name);
353	}
354
355	if (opt_forked) {
356		exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
357		return 1; /* unreachable */
358	} else {
359		return (int)outcome;
360	}
361}
362
363int
364tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag)
365{
366	int i, j;
367	size_t length = LONGEST_TEST_NAME;
368	char fullname[LONGEST_TEST_NAME];
369	int found=0;
370	if (strstr(arg, ".."))
371		length = strstr(arg,"..")-arg;
372	for (i=0; groups[i].prefix; ++i) {
373		for (j=0; groups[i].cases[j].name; ++j) {
374			struct testcase_t *testcase = &groups[i].cases[j];
375			snprintf(fullname, sizeof(fullname), "%s%s",
376				 groups[i].prefix, testcase->name);
377			if (!flag) { /* Hack! */
378				printf("    %s", fullname);
379				if (testcase->flags & TT_OFF_BY_DEFAULT)
380					puts("   (Off by default)");
381				else if (testcase->flags & TT_SKIP)
382					puts("  (DISABLED)");
383				else
384					puts("");
385			}
386			if (!strncmp(fullname, arg, length)) {
387				if (set)
388					testcase->flags |= flag;
389				else
390					testcase->flags &= ~flag;
391				++found;
392			}
393		}
394	}
395	return found;
396}
397
398static void
399usage(struct testgroup_t *groups, int list_groups)
400{
401	puts("Options are: [--verbose|--quiet|--terse] [--no-fork] [--timeout <sec>]");
402	puts("  Specify tests by name, or using a prefix ending with '..'");
403	puts("  To skip a test, prefix its name with a colon.");
404	puts("  To enable a disabled test, prefix its name with a plus.");
405	puts("  Use --list-tests for a list of tests.");
406	if (list_groups) {
407		puts("Known tests are:");
408		tinytest_set_flag_(groups, "..", 1, 0);
409	}
410	exit(0);
411}
412
413static int
414process_test_alias(struct testgroup_t *groups, const char *test)
415{
416	int i, j, n, r;
417	for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) {
418		if (!strcmp(cfg_aliases[i].name, test)) {
419			n = 0;
420			for (j = 0; cfg_aliases[i].tests[j]; ++j) {
421				r = process_test_option(groups, cfg_aliases[i].tests[j]);
422				if (r<0)
423					return -1;
424				n += r;
425			}
426			return n;
427		}
428	}
429	printf("No such test alias as @%s!",test);
430	return -1;
431}
432
433static int
434process_test_option(struct testgroup_t *groups, const char *test)
435{
436	int flag = TT_ENABLED_;
437	int n = 0;
438	if (test[0] == '@') {
439		return process_test_alias(groups, test + 1);
440	} else if (test[0] == ':') {
441		++test;
442		flag = TT_SKIP;
443	} else if (test[0] == '+') {
444		++test;
445		++n;
446		if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) {
447			printf("No such test as %s!\n", test);
448			return -1;
449		}
450	} else {
451		++n;
452	}
453	if (!tinytest_set_flag_(groups, test, 1, flag)) {
454		printf("No such test as %s!\n", test);
455		return -1;
456	}
457	return n;
458}
459
460void
461tinytest_set_aliases(const struct testlist_alias_t *aliases)
462{
463	cfg_aliases = aliases;
464}
465
466int
467tinytest_main(int c, const char **v, struct testgroup_t *groups)
468{
469	int i, j, n=0;
470
471#ifdef _WIN32
472	const char *sp = strrchr(v[0], '.');
473	const char *extension = "";
474	if (!sp || stricmp(sp, ".exe"))
475		extension = ".exe"; /* Add an exe so CreateProcess will work */
476	snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
477	commandname[MAX_PATH]='\0';
478#endif
479	for (i=1; i<c; ++i) {
480		if (v[i][0] == '-') {
481			if (!strcmp(v[i], "--RUNNING-FORKED")) {
482				opt_forked = 1;
483			} else if (!strcmp(v[i], "--no-fork")) {
484				opt_nofork = 1;
485			} else if (!strcmp(v[i], "--quiet")) {
486				opt_verbosity = -1;
487				verbosity_flag = "--quiet";
488			} else if (!strcmp(v[i], "--verbose")) {
489				opt_verbosity = 2;
490				verbosity_flag = "--verbose";
491			} else if (!strcmp(v[i], "--terse")) {
492				opt_verbosity = 0;
493				verbosity_flag = "--terse";
494			} else if (!strcmp(v[i], "--help")) {
495				usage(groups, 0);
496			} else if (!strcmp(v[i], "--list-tests")) {
497				usage(groups, 1);
498			} else if (!strcmp(v[i], "--timeout")) {
499				++i;
500				if (i >= c) {
501					fprintf(stderr, "--timeout requires argument\n");
502					return -1;
503				}
504				opt_timeout = (unsigned)atoi(v[i]);
505			} else {
506				fprintf(stderr, "Unknown option %s. Try --help\n", v[i]);
507				return -1;
508			}
509		} else {
510			int r = process_test_option(groups, v[i]);
511			if (r<0)
512				return -1;
513			n += r;
514		}
515	}
516	if (!n)
517		tinytest_set_flag_(groups, "..", 1, TT_ENABLED_);
518
519#ifdef _IONBF
520	setvbuf(stdout, NULL, _IONBF, 0);
521#endif
522
523	++in_tinytest_main;
524	for (i = 0; groups[i].prefix; ++i) {
525		struct testgroup_t *group = &groups[i];
526		for (j = 0; group->cases[j].name; ++j) {
527			struct testcase_t *testcase = &group->cases[j];
528			int test_attempts = 3;
529			int test_ret_err;
530
531			if (!(testcase->flags & TT_ENABLED_))
532				continue;
533
534			for (;;) {
535				test_ret_err = testcase_run_one(group, testcase);
536
537				if (test_ret_err == OK)
538					break;
539				if (!(testcase->flags & TT_RETRIABLE))
540					break;
541				printf("\n  [RETRYING %s (%i)]\n", testcase->name, test_attempts);
542				if (!test_attempts--)
543					break;
544			}
545
546			switch (test_ret_err) {
547				case OK:   ++n_ok;      break;
548				case SKIP: ++n_skipped; break;
549				default:   ++n_bad;     break;
550			}
551		}
552	}
553
554	--in_tinytest_main;
555
556	if (opt_verbosity==0)
557		puts("");
558
559	if (n_bad)
560		printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
561		       n_bad+n_ok,n_skipped);
562	else if (opt_verbosity >= 1)
563		printf("%d tests ok.  (%d skipped)\n", n_ok, n_skipped);
564
565	return (n_bad == 0) ? 0 : 1;
566}
567
568int
569tinytest_get_verbosity_(void)
570{
571	return opt_verbosity;
572}
573
574void
575tinytest_set_test_failed_(void)
576{
577	if (opt_verbosity <= 0 && cur_test_name) {
578		if (opt_verbosity==0) puts("");
579		printf("%s%s: ", cur_test_prefix, cur_test_name);
580		cur_test_name = NULL;
581	}
582	cur_test_outcome = FAIL;
583}
584
585void
586tinytest_set_test_skipped_(void)
587{
588	if (cur_test_outcome==OK)
589		cur_test_outcome = SKIP;
590}
591
592char *
593tinytest_format_hex_(const void *val_, unsigned long len)
594{
595	const unsigned char *val = val_;
596	char *result, *cp;
597	size_t i;
598
599	if (!val)
600		return strdup("null");
601	if (!(result = malloc(len*2+1)))
602		return strdup("<allocation failure>");
603	cp = result;
604	for (i=0;i<len;++i) {
605		*cp++ = "0123456789ABCDEF"[val[i] >> 4];
606		*cp++ = "0123456789ABCDEF"[val[i] & 0x0f];
607	}
608	*cp = 0;
609	return result;
610}
611