tests.cpp revision 260029
1// 2// Automated Testing Framework (atf) 3// 4// Copyright (c) 2007 The NetBSD Foundation, Inc. 5// All rights reserved. 6// 7// Redistribution and use in source and binary forms, with or without 8// modification, are permitted provided that the following conditions 9// are met: 10// 1. Redistributions of source code must retain the above copyright 11// notice, this list of conditions and the following disclaimer. 12// 2. Redistributions in binary form must reproduce the above copyright 13// notice, this list of conditions and the following disclaimer in the 14// documentation and/or other materials provided with the distribution. 15// 16// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND 17// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 18// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY 21// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 27// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28// 29 30extern "C" { 31#include <sys/types.h> 32#include <sys/stat.h> 33#include <sys/time.h> 34#include <sys/wait.h> 35#include <signal.h> 36#include <unistd.h> 37} 38 39#include <algorithm> 40#include <cctype> 41#include <cerrno> 42#include <cstdlib> 43#include <cstring> 44#include <fstream> 45#include <iostream> 46#include <map> 47#include <memory> 48#include <sstream> 49#include <stdexcept> 50#include <vector> 51 52extern "C" { 53#include "atf-c/error.h" 54#include "atf-c/tc.h" 55#include "atf-c/utils.h" 56} 57 58#include "noncopyable.hpp" 59#include "tests.hpp" 60 61#include "detail/application.hpp" 62#include "detail/auto_array.hpp" 63#include "detail/env.hpp" 64#include "detail/exceptions.hpp" 65#include "detail/fs.hpp" 66#include "detail/parser.hpp" 67#include "detail/sanity.hpp" 68#include "detail/text.hpp" 69 70namespace impl = atf::tests; 71namespace detail = atf::tests::detail; 72#define IMPL_NAME "atf::tests" 73 74// ------------------------------------------------------------------------ 75// The "atf_tp_writer" class. 76// ------------------------------------------------------------------------ 77 78detail::atf_tp_writer::atf_tp_writer(std::ostream& os) : 79 m_os(os), 80 m_is_first(true) 81{ 82 atf::parser::headers_map hm; 83 atf::parser::attrs_map ct_attrs; 84 ct_attrs["version"] = "1"; 85 hm["Content-Type"] = atf::parser::header_entry("Content-Type", 86 "application/X-atf-tp", ct_attrs); 87 atf::parser::write_headers(hm, m_os); 88} 89 90void 91detail::atf_tp_writer::start_tc(const std::string& ident) 92{ 93 if (!m_is_first) 94 m_os << "\n"; 95 m_os << "ident: " << ident << "\n"; 96 m_os.flush(); 97} 98 99void 100detail::atf_tp_writer::end_tc(void) 101{ 102 if (m_is_first) 103 m_is_first = false; 104} 105 106void 107detail::atf_tp_writer::tc_meta_data(const std::string& name, 108 const std::string& value) 109{ 110 PRE(name != "ident"); 111 m_os << name << ": " << value << "\n"; 112 m_os.flush(); 113} 114 115// ------------------------------------------------------------------------ 116// Free helper functions. 117// ------------------------------------------------------------------------ 118 119bool 120detail::match(const std::string& regexp, const std::string& str) 121{ 122 return atf::text::match(str, regexp); 123} 124 125// ------------------------------------------------------------------------ 126// The "tc" class. 127// ------------------------------------------------------------------------ 128 129static std::map< atf_tc_t*, impl::tc* > wraps; 130static std::map< const atf_tc_t*, const impl::tc* > cwraps; 131 132struct impl::tc_impl : atf::noncopyable { 133 std::string m_ident; 134 atf_tc_t m_tc; 135 bool m_has_cleanup; 136 137 tc_impl(const std::string& ident, const bool has_cleanup) : 138 m_ident(ident), 139 m_has_cleanup(has_cleanup) 140 { 141 } 142 143 static void 144 wrap_head(atf_tc_t *tc) 145 { 146 std::map< atf_tc_t*, impl::tc* >::iterator iter = wraps.find(tc); 147 INV(iter != wraps.end()); 148 (*iter).second->head(); 149 } 150 151 static void 152 wrap_body(const atf_tc_t *tc) 153 { 154 std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter = 155 cwraps.find(tc); 156 INV(iter != cwraps.end()); 157 try { 158 (*iter).second->body(); 159 } catch (const std::exception& e) { 160 (*iter).second->fail("Caught unhandled exception: " + std::string( 161 e.what())); 162 } catch (...) { 163 (*iter).second->fail("Caught unknown exception"); 164 } 165 } 166 167 static void 168 wrap_cleanup(const atf_tc_t *tc) 169 { 170 std::map< const atf_tc_t*, const impl::tc* >::const_iterator iter = 171 cwraps.find(tc); 172 INV(iter != cwraps.end()); 173 (*iter).second->cleanup(); 174 } 175}; 176 177impl::tc::tc(const std::string& ident, const bool has_cleanup) : 178 pimpl(new tc_impl(ident, has_cleanup)) 179{ 180} 181 182impl::tc::~tc(void) 183{ 184 cwraps.erase(&pimpl->m_tc); 185 wraps.erase(&pimpl->m_tc); 186 187 atf_tc_fini(&pimpl->m_tc); 188} 189 190void 191impl::tc::init(const vars_map& config) 192{ 193 atf_error_t err; 194 195 auto_array< const char * > array(new const char*[(config.size() * 2) + 1]); 196 const char **ptr = array.get(); 197 for (vars_map::const_iterator iter = config.begin(); 198 iter != config.end(); iter++) { 199 *ptr = (*iter).first.c_str(); 200 *(ptr + 1) = (*iter).second.c_str(); 201 ptr += 2; 202 } 203 *ptr = NULL; 204 205 wraps[&pimpl->m_tc] = this; 206 cwraps[&pimpl->m_tc] = this; 207 208 err = atf_tc_init(&pimpl->m_tc, pimpl->m_ident.c_str(), pimpl->wrap_head, 209 pimpl->wrap_body, pimpl->m_has_cleanup ? pimpl->wrap_cleanup : NULL, 210 array.get()); 211 if (atf_is_error(err)) 212 throw_atf_error(err); 213} 214 215bool 216impl::tc::has_config_var(const std::string& var) 217 const 218{ 219 return atf_tc_has_config_var(&pimpl->m_tc, var.c_str()); 220} 221 222bool 223impl::tc::has_md_var(const std::string& var) 224 const 225{ 226 return atf_tc_has_md_var(&pimpl->m_tc, var.c_str()); 227} 228 229const std::string 230impl::tc::get_config_var(const std::string& var) 231 const 232{ 233 return atf_tc_get_config_var(&pimpl->m_tc, var.c_str()); 234} 235 236const std::string 237impl::tc::get_config_var(const std::string& var, const std::string& defval) 238 const 239{ 240 return atf_tc_get_config_var_wd(&pimpl->m_tc, var.c_str(), defval.c_str()); 241} 242 243const std::string 244impl::tc::get_md_var(const std::string& var) 245 const 246{ 247 return atf_tc_get_md_var(&pimpl->m_tc, var.c_str()); 248} 249 250const impl::vars_map 251impl::tc::get_md_vars(void) 252 const 253{ 254 vars_map vars; 255 256 char **array = atf_tc_get_md_vars(&pimpl->m_tc); 257 try { 258 char **ptr; 259 for (ptr = array; *ptr != NULL; ptr += 2) 260 vars[*ptr] = *(ptr + 1); 261 } catch (...) { 262 atf_utils_free_charpp(array); 263 throw; 264 } 265 266 return vars; 267} 268 269void 270impl::tc::set_md_var(const std::string& var, const std::string& val) 271{ 272 atf_error_t err = atf_tc_set_md_var(&pimpl->m_tc, var.c_str(), val.c_str()); 273 if (atf_is_error(err)) 274 throw_atf_error(err); 275} 276 277void 278impl::tc::run(const std::string& resfile) 279 const 280{ 281 atf_error_t err = atf_tc_run(&pimpl->m_tc, resfile.c_str()); 282 if (atf_is_error(err)) 283 throw_atf_error(err); 284} 285 286void 287impl::tc::run_cleanup(void) 288 const 289{ 290 atf_error_t err = atf_tc_cleanup(&pimpl->m_tc); 291 if (atf_is_error(err)) 292 throw_atf_error(err); 293} 294 295void 296impl::tc::head(void) 297{ 298} 299 300void 301impl::tc::cleanup(void) 302 const 303{ 304} 305 306void 307impl::tc::require_prog(const std::string& prog) 308 const 309{ 310 atf_tc_require_prog(prog.c_str()); 311} 312 313void 314impl::tc::pass(void) 315{ 316 atf_tc_pass(); 317} 318 319void 320impl::tc::fail(const std::string& reason) 321{ 322 atf_tc_fail("%s", reason.c_str()); 323} 324 325void 326impl::tc::fail_nonfatal(const std::string& reason) 327{ 328 atf_tc_fail_nonfatal("%s", reason.c_str()); 329} 330 331void 332impl::tc::skip(const std::string& reason) 333{ 334 atf_tc_skip("%s", reason.c_str()); 335} 336 337void 338impl::tc::check_errno(const char* file, const int line, const int exp_errno, 339 const char* expr_str, const bool result) 340{ 341 atf_tc_check_errno(file, line, exp_errno, expr_str, result); 342} 343 344void 345impl::tc::require_errno(const char* file, const int line, const int exp_errno, 346 const char* expr_str, const bool result) 347{ 348 atf_tc_require_errno(file, line, exp_errno, expr_str, result); 349} 350 351void 352impl::tc::expect_pass(void) 353{ 354 atf_tc_expect_pass(); 355} 356 357void 358impl::tc::expect_fail(const std::string& reason) 359{ 360 atf_tc_expect_fail("%s", reason.c_str()); 361} 362 363void 364impl::tc::expect_exit(const int exitcode, const std::string& reason) 365{ 366 atf_tc_expect_exit(exitcode, "%s", reason.c_str()); 367} 368 369void 370impl::tc::expect_signal(const int signo, const std::string& reason) 371{ 372 atf_tc_expect_signal(signo, "%s", reason.c_str()); 373} 374 375void 376impl::tc::expect_death(const std::string& reason) 377{ 378 atf_tc_expect_death("%s", reason.c_str()); 379} 380 381void 382impl::tc::expect_timeout(const std::string& reason) 383{ 384 atf_tc_expect_timeout("%s", reason.c_str()); 385} 386 387// ------------------------------------------------------------------------ 388// The "tp" class. 389// ------------------------------------------------------------------------ 390 391class tp : public atf::application::app { 392public: 393 typedef std::vector< impl::tc * > tc_vector; 394 395private: 396 static const char* m_description; 397 398 bool m_lflag; 399 atf::fs::path m_resfile; 400 std::string m_srcdir_arg; 401 atf::fs::path m_srcdir; 402 403 atf::tests::vars_map m_vars; 404 405 std::string specific_args(void) const; 406 options_set specific_options(void) const; 407 void process_option(int, const char*); 408 409 void (*m_add_tcs)(tc_vector&); 410 tc_vector m_tcs; 411 412 void parse_vflag(const std::string&); 413 void handle_srcdir(void); 414 415 tc_vector init_tcs(void); 416 417 enum tc_part { 418 BODY, 419 CLEANUP, 420 }; 421 422 void list_tcs(void); 423 impl::tc* find_tc(tc_vector, const std::string&); 424 static std::pair< std::string, tc_part > process_tcarg(const std::string&); 425 int run_tc(const std::string&); 426 427public: 428 tp(void (*)(tc_vector&)); 429 ~tp(void); 430 431 int main(void); 432}; 433 434const char* tp::m_description = 435 "This is an independent atf test program."; 436 437tp::tp(void (*add_tcs)(tc_vector&)) : 438 app(m_description, "atf-test-program(1)", "atf(7)", false), 439 m_lflag(false), 440 m_resfile("/dev/stdout"), 441 m_srcdir("."), 442 m_add_tcs(add_tcs) 443{ 444} 445 446tp::~tp(void) 447{ 448 for (tc_vector::iterator iter = m_tcs.begin(); 449 iter != m_tcs.end(); iter++) { 450 impl::tc* tc = *iter; 451 452 delete tc; 453 } 454} 455 456std::string 457tp::specific_args(void) 458 const 459{ 460 return "test_case"; 461} 462 463tp::options_set 464tp::specific_options(void) 465 const 466{ 467 using atf::application::option; 468 options_set opts; 469 opts.insert(option('l', "", "List test cases and their purpose")); 470 opts.insert(option('r', "resfile", "The file to which the test program " 471 "will write the results of the " 472 "executed test case")); 473 opts.insert(option('s', "srcdir", "Directory where the test's data " 474 "files are located")); 475 opts.insert(option('v', "var=value", "Sets the configuration variable " 476 "`var' to `value'")); 477 return opts; 478} 479 480void 481tp::process_option(int ch, const char* arg) 482{ 483 switch (ch) { 484 case 'l': 485 m_lflag = true; 486 break; 487 488 case 'r': 489 m_resfile = atf::fs::path(arg); 490 break; 491 492 case 's': 493 m_srcdir_arg = arg; 494 break; 495 496 case 'v': 497 parse_vflag(arg); 498 break; 499 500 default: 501 UNREACHABLE; 502 } 503} 504 505void 506tp::parse_vflag(const std::string& str) 507{ 508 if (str.empty()) 509 throw std::runtime_error("-v requires a non-empty argument"); 510 511 std::vector< std::string > ws = atf::text::split(str, "="); 512 if (ws.size() == 1 && str[str.length() - 1] == '=') { 513 m_vars[ws[0]] = ""; 514 } else { 515 if (ws.size() != 2) 516 throw std::runtime_error("-v requires an argument of the form " 517 "var=value"); 518 519 m_vars[ws[0]] = ws[1]; 520 } 521} 522 523void 524tp::handle_srcdir(void) 525{ 526 if (m_srcdir_arg.empty()) { 527 m_srcdir = atf::fs::path(m_argv0).branch_path(); 528 if (m_srcdir.leaf_name() == ".libs") 529 m_srcdir = m_srcdir.branch_path(); 530 } else 531 m_srcdir = atf::fs::path(m_srcdir_arg); 532 533 if (!atf::fs::exists(m_srcdir / m_prog_name)) 534 throw std::runtime_error("Cannot find the test program in the " 535 "source directory `" + m_srcdir.str() + "'"); 536 537 if (!m_srcdir.is_absolute()) 538 m_srcdir = m_srcdir.to_absolute(); 539 540 m_vars["srcdir"] = m_srcdir.str(); 541} 542 543tp::tc_vector 544tp::init_tcs(void) 545{ 546 m_add_tcs(m_tcs); 547 for (tc_vector::iterator iter = m_tcs.begin(); 548 iter != m_tcs.end(); iter++) { 549 impl::tc* tc = *iter; 550 551 tc->init(m_vars); 552 } 553 return m_tcs; 554} 555 556// 557// An auxiliary unary predicate that compares the given test case's 558// identifier to the identifier stored in it. 559// 560class tc_equal_to_ident { 561 const std::string& m_ident; 562 563public: 564 tc_equal_to_ident(const std::string& i) : 565 m_ident(i) 566 { 567 } 568 569 bool operator()(const impl::tc* tc) 570 { 571 return tc->get_md_var("ident") == m_ident; 572 } 573}; 574 575void 576tp::list_tcs(void) 577{ 578 tc_vector tcs = init_tcs(); 579 detail::atf_tp_writer writer(std::cout); 580 581 for (tc_vector::const_iterator iter = tcs.begin(); 582 iter != tcs.end(); iter++) { 583 const impl::vars_map vars = (*iter)->get_md_vars(); 584 585 { 586 impl::vars_map::const_iterator iter2 = vars.find("ident"); 587 INV(iter2 != vars.end()); 588 writer.start_tc((*iter2).second); 589 } 590 591 for (impl::vars_map::const_iterator iter2 = vars.begin(); 592 iter2 != vars.end(); iter2++) { 593 const std::string& key = (*iter2).first; 594 if (key != "ident") 595 writer.tc_meta_data(key, (*iter2).second); 596 } 597 598 writer.end_tc(); 599 } 600} 601 602impl::tc* 603tp::find_tc(tc_vector tcs, const std::string& name) 604{ 605 std::vector< std::string > ids; 606 for (tc_vector::iterator iter = tcs.begin(); 607 iter != tcs.end(); iter++) { 608 impl::tc* tc = *iter; 609 610 if (tc->get_md_var("ident") == name) 611 return tc; 612 } 613 throw atf::application::usage_error("Unknown test case `%s'", 614 name.c_str()); 615} 616 617std::pair< std::string, tp::tc_part > 618tp::process_tcarg(const std::string& tcarg) 619{ 620 const std::string::size_type pos = tcarg.find(':'); 621 if (pos == std::string::npos) { 622 return std::make_pair(tcarg, BODY); 623 } else { 624 const std::string tcname = tcarg.substr(0, pos); 625 626 const std::string partname = tcarg.substr(pos + 1); 627 if (partname == "body") 628 return std::make_pair(tcname, BODY); 629 else if (partname == "cleanup") 630 return std::make_pair(tcname, CLEANUP); 631 else { 632 using atf::application::usage_error; 633 throw usage_error("Invalid test case part `%s'", partname.c_str()); 634 } 635 } 636} 637 638int 639tp::run_tc(const std::string& tcarg) 640{ 641 const std::pair< std::string, tc_part > fields = process_tcarg(tcarg); 642 643 impl::tc* tc = find_tc(init_tcs(), fields.first); 644 645 if (!atf::env::has("__RUNNING_INSIDE_ATF_RUN") || atf::env::get( 646 "__RUNNING_INSIDE_ATF_RUN") != "internal-yes-value") 647 { 648 std::cerr << m_prog_name << ": WARNING: Running test cases without " 649 "atf-run(1) is unsupported\n"; 650 std::cerr << m_prog_name << ": WARNING: No isolation nor timeout " 651 "control is being applied; you may get unexpected failures; see " 652 "atf-test-case(4)\n"; 653 } 654 655 try { 656 switch (fields.second) { 657 case BODY: 658 tc->run(m_resfile.str()); 659 break; 660 case CLEANUP: 661 tc->run_cleanup(); 662 break; 663 default: 664 UNREACHABLE; 665 } 666 return EXIT_SUCCESS; 667 } catch (const std::runtime_error& e) { 668 std::cerr << "ERROR: " << e.what() << "\n"; 669 return EXIT_FAILURE; 670 } 671} 672 673int 674tp::main(void) 675{ 676 using atf::application::usage_error; 677 678 int errcode; 679 680 handle_srcdir(); 681 682 if (m_lflag) { 683 if (m_argc > 0) 684 throw usage_error("Cannot provide test case names with -l"); 685 686 list_tcs(); 687 errcode = EXIT_SUCCESS; 688 } else { 689 if (m_argc == 0) 690 throw usage_error("Must provide a test case name"); 691 else if (m_argc > 1) 692 throw usage_error("Cannot provide more than one test case name"); 693 INV(m_argc == 1); 694 695 errcode = run_tc(m_argv[0]); 696 } 697 698 return errcode; 699} 700 701namespace atf { 702 namespace tests { 703 int run_tp(int, char* const*, void (*)(tp::tc_vector&)); 704 } 705} 706 707int 708impl::run_tp(int argc, char* const* argv, void (*add_tcs)(tp::tc_vector&)) 709{ 710 return tp(add_tcs).run(argc, argv); 711} 712