tests.cpp revision 275988 
1// Copyright (c) 2007 The NetBSD Foundation, Inc.
2// All rights reserved.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions
6// are met:
7// 1. Redistributions of source code must retain the above copyright
8//    notice, this list of conditions and the following disclaimer.
9// 2. Redistributions in binary form must reproduce the above copyright
10//    notice, this list of conditions and the following disclaimer in the
11//    documentation and/or other materials provided with the distribution.
12//
13// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
14// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
15// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
16// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
18// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
20// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
22// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
24// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26#include "atf-c++/tests.hpp"
27
28#if defined(HAVE_CONFIG_H)
29#include "config.h"
30#endif
31
32extern "C" {
33#include <sys/types.h>
34#include <sys/stat.h>
35#include <sys/time.h>
36#include <sys/wait.h>
37#include <signal.h>
38#include <unistd.h>
39}
40
41#include <algorithm>
42#include <cctype>
43#include <cerrno>
44#include <cstdlib>
45#include <cstring>
46#include <fstream>
47#include <iostream>
48#include <map>
49#include <memory>
50#include <sstream>
51#include <stdexcept>
52#include <vector>
53
54extern "C" {
55#include "atf-c/error.h"
56#include "atf-c/tc.h"
57#include "atf-c/utils.h"
58}
59
60#include "atf-c++/detail/application.hpp"
61#include "atf-c++/detail/auto_array.hpp"
62#include "atf-c++/detail/env.hpp"
63#include "atf-c++/detail/exceptions.hpp"
64#include "atf-c++/detail/fs.hpp"
65#include "atf-c++/detail/sanity.hpp"
66#include "atf-c++/detail/text.hpp"
67
68#if defined(HAVE_GNU_GETOPT)
69#   define GETOPT_POSIX "+"
70#else
71#   define GETOPT_POSIX ""
72#endif
73
74namespace impl = atf::tests;
75namespace detail = atf::tests::detail;
76#define IMPL_NAME "atf::tests"
77
78using atf::application::usage_error;
79
80// ------------------------------------------------------------------------
81// The "atf_tp_writer" class.
82// ------------------------------------------------------------------------
83
84detail::atf_tp_writer::atf_tp_writer(std::ostream& os) :
85    m_os(os),
86    m_is_first(true)
87{
88    m_os << "Content-Type: application/X-atf-tp; version=\"1\"\n\n";
89}
90
91void
92detail::atf_tp_writer::start_tc(const std::string& ident)
93{
94    if (!m_is_first)
95        m_os << "\n";
96    m_os << "ident: " << ident << "\n";
97    m_os.flush();
98}
99
100void
101detail::atf_tp_writer::end_tc(void)
102{
103    if (m_is_first)
104        m_is_first = false;
105}
106
107void
108detail::atf_tp_writer::tc_meta_data(const std::string& name,
109                                    const std::string& value)
110{
111    PRE(name != "ident");
112    m_os << name << ": " << value << "\n";
113    m_os.flush();
114}
115
116// ------------------------------------------------------------------------
117// Free helper functions.
118// ------------------------------------------------------------------------
119
120std::string Program_Name;
121
122static void
123set_program_name(const char* argv0)
124{
125    const std::string program_name = atf::fs::path(argv0).leaf_name();
126    // Libtool workaround: if running from within the source tree (binaries
127    // that are not installed yet), skip the "lt-" prefix added to files in
128    // the ".libs" directory to show the real (not temporary) name.
129    if (program_name.substr(0, 3) == "lt-")
130        Program_Name = program_name.substr(3);
131    else
132        Program_Name = program_name;
133}
134
135bool
136detail::match(const std::string& regexp, const std::string& str)
137{
138    return atf::text::match(str, regexp);
139}
140
141// ------------------------------------------------------------------------
142// The "tc" class.
143// ------------------------------------------------------------------------
144
145static std::map< atf_tc_t*, impl::tc* > wraps;
146static std::map< const atf_tc_t*, const impl::tc* > cwraps;
147
148struct impl::tc_impl {
149private:
150    // Non-copyable.
151    tc_impl(const tc_impl&);
152    tc_impl& operator=(const tc_impl&);
153
154public:
155    std::string m_ident;
156    atf_tc_t m_tc;
157    bool m_has_cleanup;
158
159    tc_impl(const std::string& ident, const bool has_cleanup) :
160        m_ident(ident),
161        m_has_cleanup(has_cleanup)
162    {
163    }
164
165    static void
166    wrap_head(atf_tc_t *tc)
167    {
168        std::map< atf_tc_t*, impl::tc* >::iterator iter = wraps.find(tc);
169        INV(iter != wraps.end());
170        (*iter).second->head();
171    }
172
173    static void
174    wrap_body(const atf_tc_t *tc)
175    {
176        std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter =
177            cwraps.find(tc);
178        INV(iter != cwraps.end());
179        (*iter).second->body();
180    }
181
182    static void
183    wrap_cleanup(const atf_tc_t *tc)
184    {
185        std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter =
186            cwraps.find(tc);
187        INV(iter != cwraps.end());
188        (*iter).second->cleanup();
189    }
190};
191
192impl::tc::tc(const std::string& ident, const bool has_cleanup) :
193    pimpl(new tc_impl(ident, has_cleanup))
194{
195}
196
197impl::tc::~tc(void)
198{
199    cwraps.erase(&pimpl->m_tc);
200    wraps.erase(&pimpl->m_tc);
201
202    atf_tc_fini(&pimpl->m_tc);
203}
204
205void
206impl::tc::init(const vars_map& config)
207{
208    atf_error_t err;
209
210    auto_array< const char * > array(new const char*[(config.size() * 2) + 1]);
211    const char **ptr = array.get();
212    for (vars_map::const_iterator iter = config.begin();
213         iter != config.end(); iter++) {
214         *ptr = (*iter).first.c_str();
215         *(ptr + 1) = (*iter).second.c_str();
216         ptr += 2;
217    }
218    *ptr = NULL;
219
220    wraps[&pimpl->m_tc] = this;
221    cwraps[&pimpl->m_tc] = this;
222
223    err = atf_tc_init(&pimpl->m_tc, pimpl->m_ident.c_str(), pimpl->wrap_head,
224        pimpl->wrap_body, pimpl->m_has_cleanup ? pimpl->wrap_cleanup : NULL,
225        array.get());
226    if (atf_is_error(err))
227        throw_atf_error(err);
228}
229
230bool
231impl::tc::has_config_var(const std::string& var)
232    const
233{
234    return atf_tc_has_config_var(&pimpl->m_tc, var.c_str());
235}
236
237bool
238impl::tc::has_md_var(const std::string& var)
239    const
240{
241    return atf_tc_has_md_var(&pimpl->m_tc, var.c_str());
242}
243
244const std::string
245impl::tc::get_config_var(const std::string& var)
246    const
247{
248    return atf_tc_get_config_var(&pimpl->m_tc, var.c_str());
249}
250
251const std::string
252impl::tc::get_config_var(const std::string& var, const std::string& defval)
253    const
254{
255    return atf_tc_get_config_var_wd(&pimpl->m_tc, var.c_str(), defval.c_str());
256}
257
258const std::string
259impl::tc::get_md_var(const std::string& var)
260    const
261{
262    return atf_tc_get_md_var(&pimpl->m_tc, var.c_str());
263}
264
265const impl::vars_map
266impl::tc::get_md_vars(void)
267    const
268{
269    vars_map vars;
270
271    char **array = atf_tc_get_md_vars(&pimpl->m_tc);
272    try {
273        char **ptr;
274        for (ptr = array; *ptr != NULL; ptr += 2)
275            vars[*ptr] = *(ptr + 1);
276    } catch (...) {
277        atf_utils_free_charpp(array);
278        throw;
279    }
280
281    return vars;
282}
283
284void
285impl::tc::set_md_var(const std::string& var, const std::string& val)
286{
287    atf_error_t err = atf_tc_set_md_var(&pimpl->m_tc, var.c_str(), val.c_str());
288    if (atf_is_error(err))
289        throw_atf_error(err);
290}
291
292void
293impl::tc::run(const std::string& resfile)
294    const
295{
296    atf_error_t err = atf_tc_run(&pimpl->m_tc, resfile.c_str());
297    if (atf_is_error(err))
298        throw_atf_error(err);
299}
300
301void
302impl::tc::run_cleanup(void)
303    const
304{
305    atf_error_t err = atf_tc_cleanup(&pimpl->m_tc);
306    if (atf_is_error(err))
307        throw_atf_error(err);
308}
309
310void
311impl::tc::head(void)
312{
313}
314
315void
316impl::tc::cleanup(void)
317    const
318{
319}
320
321void
322impl::tc::require_prog(const std::string& prog)
323    const
324{
325    atf_tc_require_prog(prog.c_str());
326}
327
328void
329impl::tc::pass(void)
330{
331    atf_tc_pass();
332}
333
334void
335impl::tc::fail(const std::string& reason)
336{
337    atf_tc_fail("%s", reason.c_str());
338}
339
340void
341impl::tc::fail_nonfatal(const std::string& reason)
342{
343    atf_tc_fail_nonfatal("%s", reason.c_str());
344}
345
346void
347impl::tc::skip(const std::string& reason)
348{
349    atf_tc_skip("%s", reason.c_str());
350}
351
352void
353impl::tc::check_errno(const char* file, const int line, const int exp_errno,
354                      const char* expr_str, const bool result)
355{
356    atf_tc_check_errno(file, line, exp_errno, expr_str, result);
357}
358
359void
360impl::tc::require_errno(const char* file, const int line, const int exp_errno,
361                        const char* expr_str, const bool result)
362{
363    atf_tc_require_errno(file, line, exp_errno, expr_str, result);
364}
365
366void
367impl::tc::expect_pass(void)
368{
369    atf_tc_expect_pass();
370}
371
372void
373impl::tc::expect_fail(const std::string& reason)
374{
375    atf_tc_expect_fail("%s", reason.c_str());
376}
377
378void
379impl::tc::expect_exit(const int exitcode, const std::string& reason)
380{
381    atf_tc_expect_exit(exitcode, "%s", reason.c_str());
382}
383
384void
385impl::tc::expect_signal(const int signo, const std::string& reason)
386{
387    atf_tc_expect_signal(signo, "%s", reason.c_str());
388}
389
390void
391impl::tc::expect_death(const std::string& reason)
392{
393    atf_tc_expect_death("%s", reason.c_str());
394}
395
396void
397impl::tc::expect_timeout(const std::string& reason)
398{
399    atf_tc_expect_timeout("%s", reason.c_str());
400}
401
402// ------------------------------------------------------------------------
403// Test program main code.
404// ------------------------------------------------------------------------
405
406namespace {
407
408typedef std::vector< impl::tc * > tc_vector;
409
410enum tc_part { BODY, CLEANUP };
411
412static void
413parse_vflag(const std::string& str, atf::tests::vars_map& vars)
414{
415    if (str.empty())
416        throw std::runtime_error("-v requires a non-empty argument");
417
418    std::vector< std::string > ws = atf::text::split(str, "=");
419    if (ws.size() == 1 && str[str.length() - 1] == '=') {
420        vars[ws[0]] = "";
421    } else {
422        if (ws.size() != 2)
423            throw std::runtime_error("-v requires an argument of the form "
424                                     "var=value");
425
426        vars[ws[0]] = ws[1];
427    }
428}
429
430static atf::fs::path
431handle_srcdir(const char* argv0, const std::string& srcdir_arg)
432{
433    atf::fs::path srcdir(".");
434
435    if (srcdir_arg.empty()) {
436        srcdir = atf::fs::path(argv0).branch_path();
437        if (srcdir.leaf_name() == ".libs")
438            srcdir = srcdir.branch_path();
439    } else
440        srcdir = atf::fs::path(srcdir_arg);
441
442    if (!atf::fs::exists(srcdir / Program_Name))
443        throw usage_error("Cannot find the test program in the source "
444                          "directory `%s'", srcdir.c_str());
445
446    if (!srcdir.is_absolute())
447        srcdir = srcdir.to_absolute();
448
449    return srcdir;
450}
451
452static void
453init_tcs(void (*add_tcs)(tc_vector&), tc_vector& tcs,
454         const atf::tests::vars_map& vars)
455{
456    add_tcs(tcs);
457    for (tc_vector::iterator iter = tcs.begin(); iter != tcs.end(); iter++) {
458        impl::tc* tc = *iter;
459
460        tc->init(vars);
461    }
462}
463
464static int
465list_tcs(const tc_vector& tcs)
466{
467    detail::atf_tp_writer writer(std::cout);
468
469    for (tc_vector::const_iterator iter = tcs.begin();
470         iter != tcs.end(); iter++) {
471        const impl::vars_map vars = (*iter)->get_md_vars();
472
473        {
474            impl::vars_map::const_iterator iter2 = vars.find("ident");
475            INV(iter2 != vars.end());
476            writer.start_tc((*iter2).second);
477        }
478
479        for (impl::vars_map::const_iterator iter2 = vars.begin();
480             iter2 != vars.end(); iter2++) {
481            const std::string& key = (*iter2).first;
482            if (key != "ident")
483                writer.tc_meta_data(key, (*iter2).second);
484        }
485
486        writer.end_tc();
487    }
488
489    return EXIT_SUCCESS;
490}
491
492static impl::tc*
493find_tc(tc_vector tcs, const std::string& name)
494{
495    std::vector< std::string > ids;
496    for (tc_vector::iterator iter = tcs.begin();
497         iter != tcs.end(); iter++) {
498        impl::tc* tc = *iter;
499
500        if (tc->get_md_var("ident") == name)
501            return tc;
502    }
503    throw usage_error("Unknown test case `%s'", name.c_str());
504}
505
506static std::pair< std::string, tc_part >
507process_tcarg(const std::string& tcarg)
508{
509    const std::string::size_type pos = tcarg.find(':');
510    if (pos == std::string::npos) {
511        return std::make_pair(tcarg, BODY);
512    } else {
513        const std::string tcname = tcarg.substr(0, pos);
514
515        const std::string partname = tcarg.substr(pos + 1);
516        if (partname == "body")
517            return std::make_pair(tcname, BODY);
518        else if (partname == "cleanup")
519            return std::make_pair(tcname, CLEANUP);
520        else {
521            throw usage_error("Invalid test case part `%s'", partname.c_str());
522        }
523    }
524}
525
526static int
527run_tc(tc_vector& tcs, const std::string& tcarg, const atf::fs::path& resfile)
528{
529    const std::pair< std::string, tc_part > fields = process_tcarg(tcarg);
530
531    impl::tc* tc = find_tc(tcs, fields.first);
532
533    if (!atf::env::has("__RUNNING_INSIDE_ATF_RUN") || atf::env::get(
534        "__RUNNING_INSIDE_ATF_RUN") != "internal-yes-value")
535    {
536        std::cerr << Program_Name << ": WARNING: Running test cases outside "
537            "of kyua(1) is unsupported\n";
538        std::cerr << Program_Name << ": WARNING: No isolation nor timeout "
539            "control is being applied; you may get unexpected failures; see "
540            "atf-test-case(4)\n";
541    }
542
543    switch (fields.second) {
544    case BODY:
545        tc->run(resfile.str());
546        break;
547    case CLEANUP:
548        tc->run_cleanup();
549        break;
550    default:
551        UNREACHABLE;
552    }
553    return EXIT_SUCCESS;
554}
555
556static int
557safe_main(int argc, char** argv, void (*add_tcs)(tc_vector&))
558{
559    const char* argv0 = argv[0];
560
561    bool lflag = false;
562    atf::fs::path resfile("/dev/stdout");
563    std::string srcdir_arg;
564    atf::tests::vars_map vars;
565
566    int ch;
567    int old_opterr;
568
569    old_opterr = opterr;
570    ::opterr = 0;
571    while ((ch = ::getopt(argc, argv, GETOPT_POSIX ":lr:s:v:")) != -1) {
572        switch (ch) {
573        case 'l':
574            lflag = true;
575            break;
576
577        case 'r':
578            resfile = atf::fs::path(::optarg);
579            break;
580
581        case 's':
582            srcdir_arg = ::optarg;
583            break;
584
585        case 'v':
586            parse_vflag(::optarg, vars);
587            break;
588
589        case ':':
590            throw usage_error("Option -%c requires an argument.", ::optopt);
591            break;
592
593        case '?':
594        default:
595            throw usage_error("Unknown option -%c.", ::optopt);
596        }
597    }
598    argc -= optind;
599    argv += optind;
600
601    // Clear getopt state just in case the test wants to use it.
602    ::opterr = old_opterr;
603    ::optind = 1;
604#if defined(HAVE_OPTRESET)
605    ::optreset = 1;
606#endif
607
608    vars["srcdir"] = handle_srcdir(argv0, srcdir_arg).str();
609
610    int errcode;
611
612    tc_vector tcs;
613    if (lflag) {
614        if (argc > 0)
615            throw usage_error("Cannot provide test case names with -l");
616
617        init_tcs(add_tcs, tcs, vars);
618        errcode = list_tcs(tcs);
619    } else {
620        if (argc == 0)
621            throw usage_error("Must provide a test case name");
622        else if (argc > 1)
623            throw usage_error("Cannot provide more than one test case name");
624        INV(argc == 1);
625
626        init_tcs(add_tcs, tcs, vars);
627        errcode = run_tc(tcs, argv[0], resfile);
628    }
629    for (tc_vector::iterator iter = tcs.begin(); iter != tcs.end(); iter++) {
630        impl::tc* tc = *iter;
631
632        delete tc;
633    }
634
635    return errcode;
636}
637
638}  // anonymous namespace
639
640namespace atf {
641    namespace tests {
642        int run_tp(int, char**, void (*)(tc_vector&));
643    }
644}
645
646int
647impl::run_tp(int argc, char** argv, void (*add_tcs)(tc_vector&))
648{
649    try {
650        set_program_name(argv[0]);
651        return ::safe_main(argc, argv, add_tcs);
652    } catch (const usage_error& e) {
653        std::cerr
654            << Program_Name << ": ERROR: " << e.what() << '\n'
655            << Program_Name << ": See atf-test-program(1) for usage details.\n";
656        return EXIT_FAILURE;
657    }
658}
659