1240116Smarcel// 2240116Smarcel// Automated Testing Framework (atf) 3240116Smarcel// 4240116Smarcel// Copyright (c) 2007 The NetBSD Foundation, Inc. 5240116Smarcel// All rights reserved. 6240116Smarcel// 7240116Smarcel// Redistribution and use in source and binary forms, with or without 8240116Smarcel// modification, are permitted provided that the following conditions 9240116Smarcel// are met: 10240116Smarcel// 1. Redistributions of source code must retain the above copyright 11240116Smarcel// notice, this list of conditions and the following disclaimer. 12240116Smarcel// 2. Redistributions in binary form must reproduce the above copyright 13240116Smarcel// notice, this list of conditions and the following disclaimer in the 14240116Smarcel// documentation and/or other materials provided with the distribution. 15240116Smarcel// 16240116Smarcel// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND 17240116Smarcel// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 18240116Smarcel// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19240116Smarcel// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20240116Smarcel// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY 21240116Smarcel// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22240116Smarcel// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23240116Smarcel// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24240116Smarcel// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25240116Smarcel// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26240116Smarcel// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 27240116Smarcel// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28240116Smarcel// 29240116Smarcel 30240116Smarcelextern "C" { 31240116Smarcel#include <sys/types.h> 32240116Smarcel#include <sys/stat.h> 33240116Smarcel#include <sys/time.h> 34240116Smarcel 35240116Smarcel#include <fcntl.h> 36240116Smarcel#include <signal.h> 37240116Smarcel#include <unistd.h> 38240116Smarcel} 39240116Smarcel 40240116Smarcel#include <cerrno> 41240116Smarcel#include <cstdlib> 42240116Smarcel#include <cstring> 43240116Smarcel#include <fstream> 44240116Smarcel#include <iostream> 45240116Smarcel 46240116Smarcel#include "atf-c/defs.h" 47240116Smarcel 48240116Smarcel#include "atf-c++/detail/env.hpp" 49240116Smarcel#include "atf-c++/detail/parser.hpp" 50240116Smarcel#include "atf-c++/detail/process.hpp" 51240116Smarcel#include "atf-c++/detail/sanity.hpp" 52240116Smarcel#include "atf-c++/detail/text.hpp" 53240116Smarcel 54240116Smarcel#include "config.hpp" 55240116Smarcel#include "fs.hpp" 56240116Smarcel#include "io.hpp" 57240116Smarcel#include "requirements.hpp" 58240116Smarcel#include "signals.hpp" 59240116Smarcel#include "test-program.hpp" 60240116Smarcel#include "timer.hpp" 61240116Smarcel#include "user.hpp" 62240116Smarcel 63240116Smarcelnamespace impl = atf::atf_run; 64240116Smarcelnamespace detail = atf::atf_run::detail; 65240116Smarcel 66240116Smarcelnamespace { 67240116Smarcel 68240116Smarcelstatic void 69240116Smarcelcheck_stream(std::ostream& os) 70240116Smarcel{ 71240116Smarcel // If we receive a signal while writing to the stream, the bad bit gets set. 72240116Smarcel // Things seem to behave fine afterwards if we clear such error condition. 73240116Smarcel // However, I'm not sure if it's safe to query errno at this point. 74240116Smarcel if (os.bad()) { 75240116Smarcel if (errno == EINTR) 76240116Smarcel os.clear(); 77240116Smarcel else 78240116Smarcel throw std::runtime_error("Failed"); 79240116Smarcel } 80240116Smarcel} 81240116Smarcel 82240116Smarcelnamespace atf_tp { 83240116Smarcel 84240116Smarcelstatic const atf::parser::token_type eof_type = 0; 85240116Smarcelstatic const atf::parser::token_type nl_type = 1; 86240116Smarcelstatic const atf::parser::token_type text_type = 2; 87240116Smarcelstatic const atf::parser::token_type colon_type = 3; 88240116Smarcelstatic const atf::parser::token_type dblquote_type = 4; 89240116Smarcel 90240116Smarcelclass tokenizer : public atf::parser::tokenizer< std::istream > { 91240116Smarcelpublic: 92240116Smarcel tokenizer(std::istream& is, size_t curline) : 93240116Smarcel atf::parser::tokenizer< std::istream > 94240116Smarcel (is, true, eof_type, nl_type, text_type, curline) 95240116Smarcel { 96240116Smarcel add_delim(':', colon_type); 97240116Smarcel add_quote('"', dblquote_type); 98240116Smarcel } 99240116Smarcel}; 100240116Smarcel 101240116Smarcel} // namespace atf_tp 102240116Smarcel 103240116Smarcelclass metadata_reader : public detail::atf_tp_reader { 104240116Smarcel impl::test_cases_map m_tcs; 105240116Smarcel 106240116Smarcel void got_tc(const std::string& ident, const atf::tests::vars_map& props) 107240116Smarcel { 108240116Smarcel if (m_tcs.find(ident) != m_tcs.end()) 109240116Smarcel throw(std::runtime_error("Duplicate test case " + ident + 110240116Smarcel " in test program")); 111240116Smarcel m_tcs[ident] = props; 112240116Smarcel 113240116Smarcel if (m_tcs[ident].find("has.cleanup") == m_tcs[ident].end()) 114240116Smarcel m_tcs[ident].insert(std::make_pair("has.cleanup", "false")); 115240116Smarcel 116240116Smarcel if (m_tcs[ident].find("timeout") == m_tcs[ident].end()) 117240116Smarcel m_tcs[ident].insert(std::make_pair("timeout", "300")); 118240116Smarcel } 119240116Smarcel 120240116Smarcelpublic: 121240116Smarcel metadata_reader(std::istream& is) : 122240116Smarcel detail::atf_tp_reader(is) 123240116Smarcel { 124240116Smarcel } 125240116Smarcel 126240116Smarcel const impl::test_cases_map& 127240116Smarcel get_tcs(void) 128240116Smarcel const 129240116Smarcel { 130240116Smarcel return m_tcs; 131240116Smarcel } 132240116Smarcel}; 133240116Smarcel 134240116Smarcelstruct get_metadata_params { 135240116Smarcel const atf::fs::path& executable; 136240116Smarcel const atf::tests::vars_map& config; 137240116Smarcel 138240116Smarcel get_metadata_params(const atf::fs::path& p_executable, 139240116Smarcel const atf::tests::vars_map& p_config) : 140240116Smarcel executable(p_executable), 141240116Smarcel config(p_config) 142240116Smarcel { 143240116Smarcel } 144240116Smarcel}; 145240116Smarcel 146240116Smarcelstruct test_case_params { 147240116Smarcel const atf::fs::path& executable; 148240116Smarcel const std::string& test_case_name; 149240116Smarcel const std::string& test_case_part; 150240116Smarcel const atf::tests::vars_map& metadata; 151240116Smarcel const atf::tests::vars_map& config; 152240116Smarcel const atf::fs::path& resfile; 153240116Smarcel const atf::fs::path& workdir; 154240116Smarcel 155240116Smarcel test_case_params(const atf::fs::path& p_executable, 156240116Smarcel const std::string& p_test_case_name, 157240116Smarcel const std::string& p_test_case_part, 158240116Smarcel const atf::tests::vars_map& p_metadata, 159240116Smarcel const atf::tests::vars_map& p_config, 160240116Smarcel const atf::fs::path& p_resfile, 161240116Smarcel const atf::fs::path& p_workdir) : 162240116Smarcel executable(p_executable), 163240116Smarcel test_case_name(p_test_case_name), 164240116Smarcel test_case_part(p_test_case_part), 165240116Smarcel metadata(p_metadata), 166240116Smarcel config(p_config), 167240116Smarcel resfile(p_resfile), 168240116Smarcel workdir(p_workdir) 169240116Smarcel { 170240116Smarcel } 171240116Smarcel}; 172240116Smarcel 173240116Smarcelstatic 174240116Smarcelstd::string 175240116Smarcelgenerate_timestamp(void) 176240116Smarcel{ 177240116Smarcel struct timeval tv; 178240116Smarcel if (gettimeofday(&tv, NULL) == -1) 179240116Smarcel return "0.0"; 180240116Smarcel 181240116Smarcel char buf[32]; 182240116Smarcel const int len = snprintf(buf, sizeof(buf), "%ld.%ld", 183240116Smarcel static_cast< long >(tv.tv_sec), 184240116Smarcel static_cast< long >(tv.tv_usec)); 185240116Smarcel if (len >= static_cast< int >(sizeof(buf)) || len < 0) 186240116Smarcel return "0.0"; 187240116Smarcel else 188240116Smarcel return buf; 189240116Smarcel} 190240116Smarcel 191240116Smarcelstatic 192240116Smarcelvoid 193240116Smarcelappend_to_vector(std::vector< std::string >& v1, 194240116Smarcel const std::vector< std::string >& v2) 195240116Smarcel{ 196240116Smarcel std::copy(v2.begin(), v2.end(), 197240116Smarcel std::back_insert_iterator< std::vector< std::string > >(v1)); 198240116Smarcel} 199240116Smarcel 200240116Smarcelstatic 201240116Smarcelchar** 202240116Smarcelvector_to_argv(const std::vector< std::string >& v) 203240116Smarcel{ 204240116Smarcel char** argv = new char*[v.size() + 1]; 205240116Smarcel for (std::vector< std::string >::size_type i = 0; i < v.size(); i++) { 206240116Smarcel argv[i] = strdup(v[i].c_str()); 207240116Smarcel } 208240116Smarcel argv[v.size()] = NULL; 209240116Smarcel return argv; 210240116Smarcel} 211240116Smarcel 212240116Smarcelstatic 213240116Smarcelvoid 214240116Smarcelexec_or_exit(const atf::fs::path& executable, 215240116Smarcel const std::vector< std::string >& argv) 216240116Smarcel{ 217240116Smarcel // This leaks memory in case of a failure, but it is OK. Exiting will 218240116Smarcel // do the necessary cleanup. 219240116Smarcel char* const* native_argv = vector_to_argv(argv); 220240116Smarcel 221240116Smarcel ::execv(executable.c_str(), native_argv); 222240116Smarcel 223240116Smarcel const std::string message = "Failed to execute '" + executable.str() + 224240116Smarcel "': " + std::strerror(errno) + "\n"; 225240116Smarcel if (::write(STDERR_FILENO, message.c_str(), message.length()) == -1) 226240116Smarcel std::abort(); 227240116Smarcel std::exit(EXIT_FAILURE); 228240116Smarcel} 229240116Smarcel 230240116Smarcelstatic 231240116Smarcelstd::vector< std::string > 232240116Smarcelconfig_to_args(const atf::tests::vars_map& config) 233240116Smarcel{ 234240116Smarcel std::vector< std::string > args; 235240116Smarcel 236240116Smarcel for (atf::tests::vars_map::const_iterator iter = config.begin(); 237240116Smarcel iter != config.end(); iter++) 238240116Smarcel args.push_back("-v" + (*iter).first + "=" + (*iter).second); 239240116Smarcel 240240116Smarcel return args; 241240116Smarcel} 242240116Smarcel 243240116Smarcelstatic 244240116Smarcelvoid 245240116Smarcelsilence_stdin(void) 246240116Smarcel{ 247240116Smarcel ::close(STDIN_FILENO); 248240116Smarcel int fd = ::open("/dev/null", O_RDONLY); 249240116Smarcel if (fd == -1) 250240116Smarcel throw std::runtime_error("Could not open /dev/null"); 251240116Smarcel INV(fd == STDIN_FILENO); 252240116Smarcel} 253240116Smarcel 254240116Smarcelstatic 255240116Smarcelvoid 256240116Smarcelprepare_child(const atf::fs::path& workdir) 257240116Smarcel{ 258240116Smarcel const int ret = ::setpgid(::getpid(), 0); 259240116Smarcel INV(ret != -1); 260240116Smarcel 261240116Smarcel ::umask(S_IWGRP | S_IWOTH); 262240116Smarcel 263240116Smarcel for (int i = 1; i <= impl::last_signo; i++) 264240116Smarcel impl::reset(i); 265240116Smarcel 266240116Smarcel atf::env::set("HOME", workdir.str()); 267240116Smarcel atf::env::unset("LANG"); 268240116Smarcel atf::env::unset("LC_ALL"); 269240116Smarcel atf::env::unset("LC_COLLATE"); 270240116Smarcel atf::env::unset("LC_CTYPE"); 271240116Smarcel atf::env::unset("LC_MESSAGES"); 272240116Smarcel atf::env::unset("LC_MONETARY"); 273240116Smarcel atf::env::unset("LC_NUMERIC"); 274240116Smarcel atf::env::unset("LC_TIME"); 275240116Smarcel atf::env::set("TZ", "UTC"); 276240116Smarcel 277240116Smarcel atf::env::set("__RUNNING_INSIDE_ATF_RUN", "internal-yes-value"); 278240116Smarcel 279240116Smarcel impl::change_directory(workdir); 280240116Smarcel 281240116Smarcel silence_stdin(); 282240116Smarcel} 283240116Smarcel 284240116Smarcelstatic 285240116Smarcelvoid 286240116Smarcelget_metadata_child(void* raw_params) 287240116Smarcel{ 288240116Smarcel const get_metadata_params* params = 289240116Smarcel static_cast< const get_metadata_params* >(raw_params); 290240116Smarcel 291240116Smarcel std::vector< std::string > argv; 292240116Smarcel argv.push_back(params->executable.leaf_name()); 293240116Smarcel argv.push_back("-l"); 294240116Smarcel argv.push_back("-s" + params->executable.branch_path().str()); 295240116Smarcel append_to_vector(argv, config_to_args(params->config)); 296240116Smarcel 297240116Smarcel exec_or_exit(params->executable, argv); 298240116Smarcel} 299240116Smarcel 300240116Smarcelvoid 301240116Smarcelrun_test_case_child(void* raw_params) 302240116Smarcel{ 303240116Smarcel const test_case_params* params = 304240116Smarcel static_cast< const test_case_params* >(raw_params); 305240116Smarcel 306240116Smarcel const std::pair< int, int > user = impl::get_required_user( 307240116Smarcel params->metadata, params->config); 308240116Smarcel if (user.first != -1 && user.second != -1) 309240116Smarcel impl::drop_privileges(user); 310240116Smarcel 311240116Smarcel // The input 'tp' parameter may be relative and become invalid once 312240116Smarcel // we change the current working directory. 313240116Smarcel const atf::fs::path absolute_executable = params->executable.to_absolute(); 314240116Smarcel 315240116Smarcel // Prepare the test program's arguments. We use dynamic memory and 316240116Smarcel // do not care to release it. We are going to die anyway very soon, 317240116Smarcel // either due to exec(2) or to exit(3). 318240116Smarcel std::vector< std::string > argv; 319240116Smarcel argv.push_back(absolute_executable.leaf_name()); 320240116Smarcel argv.push_back("-r" + params->resfile.str()); 321240116Smarcel argv.push_back("-s" + absolute_executable.branch_path().str()); 322240116Smarcel append_to_vector(argv, config_to_args(params->config)); 323240116Smarcel argv.push_back(params->test_case_name + ":" + params->test_case_part); 324240116Smarcel 325240116Smarcel prepare_child(params->workdir); 326240116Smarcel exec_or_exit(absolute_executable, argv); 327240116Smarcel} 328240116Smarcel 329240116Smarcelstatic void 330240116Smarceltokenize_result(const std::string& line, std::string& out_state, 331240116Smarcel std::string& out_arg, std::string& out_reason) 332240116Smarcel{ 333240116Smarcel const std::string::size_type pos = line.find_first_of(":("); 334240116Smarcel if (pos == std::string::npos) { 335240116Smarcel out_state = line; 336240116Smarcel out_arg = ""; 337240116Smarcel out_reason = ""; 338240116Smarcel } else if (line[pos] == ':') { 339240116Smarcel out_state = line.substr(0, pos); 340240116Smarcel out_arg = ""; 341240116Smarcel out_reason = atf::text::trim(line.substr(pos + 1)); 342240116Smarcel } else if (line[pos] == '(') { 343240116Smarcel const std::string::size_type pos2 = line.find("):", pos); 344240116Smarcel if (pos2 == std::string::npos) 345240116Smarcel throw std::runtime_error("Invalid test case result '" + line + 346240116Smarcel "': unclosed optional argument"); 347240116Smarcel out_state = line.substr(0, pos); 348240116Smarcel out_arg = line.substr(pos + 1, pos2 - pos - 1); 349240116Smarcel out_reason = atf::text::trim(line.substr(pos2 + 2)); 350240116Smarcel } else 351240116Smarcel UNREACHABLE; 352240116Smarcel} 353240116Smarcel 354240116Smarcelstatic impl::test_case_result 355240116Smarcelhandle_result(const std::string& state, const std::string& arg, 356240116Smarcel const std::string& reason) 357240116Smarcel{ 358240116Smarcel PRE(state == "passed"); 359240116Smarcel 360240116Smarcel if (!arg.empty() || !reason.empty()) 361240116Smarcel throw std::runtime_error("The test case result '" + state + "' cannot " 362240116Smarcel "be accompanied by a reason nor an expected value"); 363240116Smarcel 364240116Smarcel return impl::test_case_result(state, -1, reason); 365240116Smarcel} 366240116Smarcel 367240116Smarcelstatic impl::test_case_result 368240116Smarcelhandle_result_with_reason(const std::string& state, const std::string& arg, 369240116Smarcel const std::string& reason) 370240116Smarcel{ 371240116Smarcel PRE(state == "expected_death" || state == "expected_failure" || 372240116Smarcel state == "expected_timeout" || state == "failed" || state == "skipped"); 373240116Smarcel 374240116Smarcel if (!arg.empty() || reason.empty()) 375240116Smarcel throw std::runtime_error("The test case result '" + state + "' must " 376240116Smarcel "be accompanied by a reason but not by an expected value"); 377240116Smarcel 378240116Smarcel return impl::test_case_result(state, -1, reason); 379240116Smarcel} 380240116Smarcel 381240116Smarcelstatic impl::test_case_result 382240116Smarcelhandle_result_with_reason_and_arg(const std::string& state, 383240116Smarcel const std::string& arg, 384240116Smarcel const std::string& reason) 385240116Smarcel{ 386240116Smarcel PRE(state == "expected_exit" || state == "expected_signal"); 387240116Smarcel 388240116Smarcel if (reason.empty()) 389240116Smarcel throw std::runtime_error("The test case result '" + state + "' must " 390240116Smarcel "be accompanied by a reason"); 391240116Smarcel 392240116Smarcel int value; 393240116Smarcel if (arg.empty()) { 394240116Smarcel value = -1; 395240116Smarcel } else { 396240116Smarcel try { 397240116Smarcel value = atf::text::to_type< int >(arg); 398240116Smarcel } catch (const std::runtime_error&) { 399240116Smarcel throw std::runtime_error("The value '" + arg + "' passed to the '" + 400240116Smarcel state + "' state must be an integer"); 401240116Smarcel } 402240116Smarcel } 403240116Smarcel 404240116Smarcel return impl::test_case_result(state, value, reason); 405240116Smarcel} 406240116Smarcel 407240116Smarcel} // anonymous namespace 408240116Smarcel 409240116Smarceldetail::atf_tp_reader::atf_tp_reader(std::istream& is) : 410240116Smarcel m_is(is) 411240116Smarcel{ 412240116Smarcel} 413240116Smarcel 414240116Smarceldetail::atf_tp_reader::~atf_tp_reader(void) 415240116Smarcel{ 416240116Smarcel} 417240116Smarcel 418240116Smarcelvoid 419240116Smarceldetail::atf_tp_reader::got_tc( 420240116Smarcel const std::string& ident ATF_DEFS_ATTRIBUTE_UNUSED, 421240116Smarcel const std::map< std::string, std::string >& md ATF_DEFS_ATTRIBUTE_UNUSED) 422240116Smarcel{ 423240116Smarcel} 424240116Smarcel 425240116Smarcelvoid 426240116Smarceldetail::atf_tp_reader::got_eof(void) 427240116Smarcel{ 428240116Smarcel} 429240116Smarcel 430240116Smarcelvoid 431240116Smarceldetail::atf_tp_reader::validate_and_insert(const std::string& name, 432240116Smarcel const std::string& value, const size_t lineno, 433240116Smarcel std::map< std::string, std::string >& md) 434240116Smarcel{ 435240116Smarcel using atf::parser::parse_error; 436240116Smarcel 437240116Smarcel if (value.empty()) 438240116Smarcel throw parse_error(lineno, "The value for '" + name +"' cannot be " 439240116Smarcel "empty"); 440240116Smarcel 441240116Smarcel const std::string ident_regex = "^[_A-Za-z0-9]+$"; 442240116Smarcel const std::string integer_regex = "^[0-9]+$"; 443240116Smarcel 444240116Smarcel if (name == "descr") { 445240116Smarcel // Any non-empty value is valid. 446240116Smarcel } else if (name == "has.cleanup") { 447240116Smarcel try { 448240116Smarcel (void)atf::text::to_bool(value); 449240116Smarcel } catch (const std::runtime_error&) { 450240116Smarcel throw parse_error(lineno, "The has.cleanup property requires a" 451240116Smarcel " boolean value"); 452240116Smarcel } 453240116Smarcel } else if (name == "ident") { 454240116Smarcel if (!atf::text::match(value, ident_regex)) 455240116Smarcel throw parse_error(lineno, "The identifier must match " + 456240116Smarcel ident_regex + "; was '" + value + "'"); 457240116Smarcel } else if (name == "require.arch") { 458240116Smarcel } else if (name == "require.config") { 459240116Smarcel } else if (name == "require.files") { 460240116Smarcel } else if (name == "require.machine") { 461240116Smarcel } else if (name == "require.memory") { 462240116Smarcel try { 463240116Smarcel (void)atf::text::to_bytes(value); 464240116Smarcel } catch (const std::runtime_error&) { 465240116Smarcel throw parse_error(lineno, "The require.memory property requires an " 466240116Smarcel "integer value representing an amount of bytes"); 467240116Smarcel } 468240116Smarcel } else if (name == "require.progs") { 469240116Smarcel } else if (name == "require.user") { 470240116Smarcel } else if (name == "timeout") { 471240116Smarcel if (!atf::text::match(value, integer_regex)) 472240116Smarcel throw parse_error(lineno, "The timeout property requires an integer" 473240116Smarcel " value"); 474240116Smarcel } else if (name == "use.fs") { 475240116Smarcel // Deprecated; ignore it. 476240116Smarcel } else if (name.size() > 2 && name[0] == 'X' && name[1] == '-') { 477240116Smarcel // Any non-empty value is valid. 478240116Smarcel } else { 479240116Smarcel throw parse_error(lineno, "Unknown property '" + name + "'"); 480240116Smarcel } 481240116Smarcel 482240116Smarcel md.insert(std::make_pair(name, value)); 483240116Smarcel} 484240116Smarcel 485240116Smarcelvoid 486240116Smarceldetail::atf_tp_reader::read(void) 487240116Smarcel{ 488240116Smarcel using atf::parser::parse_error; 489240116Smarcel using namespace atf_tp; 490240116Smarcel 491240116Smarcel std::pair< size_t, atf::parser::headers_map > hml = 492240116Smarcel atf::parser::read_headers(m_is, 1); 493240116Smarcel atf::parser::validate_content_type(hml.second, 494240116Smarcel "application/X-atf-tp", 1); 495240116Smarcel 496240116Smarcel tokenizer tkz(m_is, hml.first); 497240116Smarcel atf::parser::parser< tokenizer > p(tkz); 498240116Smarcel 499240116Smarcel try { 500240116Smarcel atf::parser::token t = p.expect(text_type, "property name"); 501240116Smarcel if (t.text() != "ident") 502240116Smarcel throw parse_error(t.lineno(), "First property of a test case " 503240116Smarcel "must be 'ident'"); 504240116Smarcel 505240116Smarcel std::map< std::string, std::string > props; 506240116Smarcel do { 507240116Smarcel const std::string name = t.text(); 508240116Smarcel t = p.expect(colon_type, "`:'"); 509240116Smarcel const std::string value = atf::text::trim(p.rest_of_line()); 510240116Smarcel t = p.expect(nl_type, "new line"); 511240116Smarcel validate_and_insert(name, value, t.lineno(), props); 512240116Smarcel 513240116Smarcel t = p.expect(eof_type, nl_type, text_type, "property name, new " 514240116Smarcel "line or eof"); 515240116Smarcel if (t.type() == nl_type || t.type() == eof_type) { 516240116Smarcel const std::map< std::string, std::string >::const_iterator 517240116Smarcel iter = props.find("ident"); 518240116Smarcel if (iter == props.end()) 519240116Smarcel throw parse_error(t.lineno(), "Test case definition did " 520240116Smarcel "not define an 'ident' property"); 521240116Smarcel ATF_PARSER_CALLBACK(p, got_tc((*iter).second, props)); 522240116Smarcel props.clear(); 523240116Smarcel 524240116Smarcel if (t.type() == nl_type) { 525240116Smarcel t = p.expect(text_type, "property name"); 526240116Smarcel if (t.text() != "ident") 527240116Smarcel throw parse_error(t.lineno(), "First property of a " 528240116Smarcel "test case must be 'ident'"); 529240116Smarcel } 530240116Smarcel } 531240116Smarcel } while (t.type() != eof_type); 532240116Smarcel ATF_PARSER_CALLBACK(p, got_eof()); 533240116Smarcel } catch (const parse_error& pe) { 534240116Smarcel p.add_error(pe); 535240116Smarcel p.reset(nl_type); 536240116Smarcel } 537240116Smarcel} 538240116Smarcel 539240116Smarcelimpl::test_case_result 540240116Smarceldetail::parse_test_case_result(const std::string& line) 541240116Smarcel{ 542240116Smarcel std::string state, arg, reason; 543240116Smarcel tokenize_result(line, state, arg, reason); 544240116Smarcel 545240116Smarcel if (state == "expected_death") 546240116Smarcel return handle_result_with_reason(state, arg, reason); 547240116Smarcel else if (state.compare(0, 13, "expected_exit") == 0) 548240116Smarcel return handle_result_with_reason_and_arg(state, arg, reason); 549240116Smarcel else if (state.compare(0, 16, "expected_failure") == 0) 550240116Smarcel return handle_result_with_reason(state, arg, reason); 551240116Smarcel else if (state.compare(0, 15, "expected_signal") == 0) 552240116Smarcel return handle_result_with_reason_and_arg(state, arg, reason); 553240116Smarcel else if (state.compare(0, 16, "expected_timeout") == 0) 554240116Smarcel return handle_result_with_reason(state, arg, reason); 555240116Smarcel else if (state == "failed") 556240116Smarcel return handle_result_with_reason(state, arg, reason); 557240116Smarcel else if (state == "passed") 558240116Smarcel return handle_result(state, arg, reason); 559240116Smarcel else if (state == "skipped") 560240116Smarcel return handle_result_with_reason(state, arg, reason); 561240116Smarcel else 562240116Smarcel throw std::runtime_error("Unknown test case result type in: " + line); 563240116Smarcel} 564240116Smarcel 565240116Smarcelimpl::atf_tps_writer::atf_tps_writer(std::ostream& os) : 566240116Smarcel m_os(os) 567240116Smarcel{ 568240116Smarcel atf::parser::headers_map hm; 569240116Smarcel atf::parser::attrs_map ct_attrs; 570240116Smarcel ct_attrs["version"] = "3"; 571240116Smarcel hm["Content-Type"] = 572240116Smarcel atf::parser::header_entry("Content-Type", "application/X-atf-tps", 573240116Smarcel ct_attrs); 574240116Smarcel atf::parser::write_headers(hm, m_os); 575240116Smarcel} 576240116Smarcel 577240116Smarcelvoid 578240116Smarcelimpl::atf_tps_writer::info(const std::string& what, const std::string& val) 579240116Smarcel{ 580240116Smarcel m_os << "info: " << what << ", " << val << "\n"; 581240116Smarcel m_os.flush(); 582240116Smarcel} 583240116Smarcel 584240116Smarcelvoid 585240116Smarcelimpl::atf_tps_writer::ntps(size_t p_ntps) 586240116Smarcel{ 587240116Smarcel m_os << "tps-count: " << p_ntps << "\n"; 588240116Smarcel m_os.flush(); 589240116Smarcel} 590240116Smarcel 591240116Smarcelvoid 592240116Smarcelimpl::atf_tps_writer::start_tp(const std::string& tp, size_t ntcs) 593240116Smarcel{ 594240116Smarcel m_tpname = tp; 595240116Smarcel m_os << "tp-start: " << generate_timestamp() << ", " << tp << ", " 596240116Smarcel << ntcs << "\n"; 597240116Smarcel m_os.flush(); 598240116Smarcel} 599240116Smarcel 600240116Smarcelvoid 601240116Smarcelimpl::atf_tps_writer::end_tp(const std::string& reason) 602240116Smarcel{ 603240116Smarcel PRE(reason.find('\n') == std::string::npos); 604240116Smarcel if (reason.empty()) 605240116Smarcel m_os << "tp-end: " << generate_timestamp() << ", " << m_tpname << "\n"; 606240116Smarcel else 607240116Smarcel m_os << "tp-end: " << generate_timestamp() << ", " << m_tpname 608240116Smarcel << ", " << reason << "\n"; 609240116Smarcel m_os.flush(); 610240116Smarcel} 611240116Smarcel 612240116Smarcelvoid 613240116Smarcelimpl::atf_tps_writer::start_tc(const std::string& tcname) 614240116Smarcel{ 615240116Smarcel m_tcname = tcname; 616240116Smarcel m_os << "tc-start: " << generate_timestamp() << ", " << tcname << "\n"; 617240116Smarcel m_os.flush(); 618240116Smarcel} 619240116Smarcel 620240116Smarcelvoid 621240116Smarcelimpl::atf_tps_writer::stdout_tc(const std::string& line) 622240116Smarcel{ 623240116Smarcel m_os << "tc-so:" << line << "\n"; 624240116Smarcel check_stream(m_os); 625240116Smarcel m_os.flush(); 626240116Smarcel check_stream(m_os); 627240116Smarcel} 628240116Smarcel 629240116Smarcelvoid 630240116Smarcelimpl::atf_tps_writer::stderr_tc(const std::string& line) 631240116Smarcel{ 632240116Smarcel m_os << "tc-se:" << line << "\n"; 633240116Smarcel check_stream(m_os); 634240116Smarcel m_os.flush(); 635240116Smarcel check_stream(m_os); 636240116Smarcel} 637240116Smarcel 638240116Smarcelvoid 639240116Smarcelimpl::atf_tps_writer::end_tc(const std::string& state, 640240116Smarcel const std::string& reason) 641240116Smarcel{ 642240116Smarcel std::string str = ", " + m_tcname + ", " + state; 643240116Smarcel if (!reason.empty()) 644240116Smarcel str += ", " + reason; 645240116Smarcel m_os << "tc-end: " << generate_timestamp() << str << "\n"; 646240116Smarcel m_os.flush(); 647240116Smarcel} 648240116Smarcel 649240116Smarcelimpl::metadata 650240116Smarcelimpl::get_metadata(const atf::fs::path& executable, 651240116Smarcel const atf::tests::vars_map& config) 652240116Smarcel{ 653240116Smarcel get_metadata_params params(executable, config); 654240116Smarcel atf::process::child child = 655240116Smarcel atf::process::fork(get_metadata_child, 656240116Smarcel atf::process::stream_capture(), 657240116Smarcel atf::process::stream_inherit(), 658240116Smarcel static_cast< void * >(¶ms)); 659240116Smarcel 660240116Smarcel impl::pistream outin(child.stdout_fd()); 661240116Smarcel 662240116Smarcel metadata_reader parser(outin); 663240116Smarcel parser.read(); 664240116Smarcel 665240116Smarcel const atf::process::status status = child.wait(); 666240116Smarcel if (!status.exited() || status.exitstatus() != EXIT_SUCCESS) 667240116Smarcel throw atf::parser::format_error("Test program returned failure " 668240116Smarcel "exit status for test case list"); 669240116Smarcel 670240116Smarcel return metadata(parser.get_tcs()); 671240116Smarcel} 672240116Smarcel 673240116Smarcelimpl::test_case_result 674240116Smarcelimpl::read_test_case_result(const atf::fs::path& results_path) 675240116Smarcel{ 676240116Smarcel std::ifstream results_file(results_path.c_str()); 677240116Smarcel if (!results_file) 678240116Smarcel throw std::runtime_error("Failed to open " + results_path.str()); 679240116Smarcel 680240116Smarcel std::string line, extra_line; 681240116Smarcel std::getline(results_file, line); 682240116Smarcel if (!results_file.good()) 683240116Smarcel throw std::runtime_error("Results file is empty"); 684240116Smarcel 685240116Smarcel while (std::getline(results_file, extra_line).good()) 686240116Smarcel line += "<<NEWLINE UNEXPECTED>>" + extra_line; 687240116Smarcel 688240116Smarcel results_file.close(); 689240116Smarcel 690240116Smarcel return detail::parse_test_case_result(line); 691240116Smarcel} 692240116Smarcel 693240116Smarcelnamespace { 694240116Smarcel 695240116Smarcelstatic volatile bool terminate_poll; 696240116Smarcel 697240116Smarcelstatic void 698240116Smarcelsigchld_handler(const int signo ATF_DEFS_ATTRIBUTE_UNUSED) 699240116Smarcel{ 700240116Smarcel terminate_poll = true; 701240116Smarcel} 702240116Smarcel 703240116Smarcelclass child_muxer : public impl::muxer { 704240116Smarcel impl::atf_tps_writer& m_writer; 705240116Smarcel 706240116Smarcel void 707240116Smarcel line_callback(const size_t index, const std::string& line) 708240116Smarcel { 709240116Smarcel switch (index) { 710240116Smarcel case 0: m_writer.stdout_tc(line); break; 711240116Smarcel case 1: m_writer.stderr_tc(line); break; 712240116Smarcel default: UNREACHABLE; 713240116Smarcel } 714240116Smarcel } 715240116Smarcel 716240116Smarcelpublic: 717240116Smarcel child_muxer(const int* fds, const size_t nfds, 718240116Smarcel impl::atf_tps_writer& writer) : 719240116Smarcel muxer(fds, nfds), 720240116Smarcel m_writer(writer) 721240116Smarcel { 722240116Smarcel } 723240116Smarcel}; 724240116Smarcel 725240116Smarcel} // anonymous namespace 726240116Smarcel 727240116Smarcelstd::pair< std::string, atf::process::status > 728240116Smarcelimpl::run_test_case(const atf::fs::path& executable, 729240116Smarcel const std::string& test_case_name, 730240116Smarcel const std::string& test_case_part, 731240116Smarcel const atf::tests::vars_map& metadata, 732240116Smarcel const atf::tests::vars_map& config, 733240116Smarcel const atf::fs::path& resfile, 734240116Smarcel const atf::fs::path& workdir, 735240116Smarcel atf_tps_writer& writer) 736240116Smarcel{ 737240116Smarcel // TODO: Capture termination signals and deliver them to the subprocess 738240116Smarcel // instead. Or maybe do something else; think about it. 739240116Smarcel 740240116Smarcel test_case_params params(executable, test_case_name, test_case_part, 741240116Smarcel metadata, config, resfile, workdir); 742240116Smarcel atf::process::child child = 743240116Smarcel atf::process::fork(run_test_case_child, 744240116Smarcel atf::process::stream_capture(), 745240116Smarcel atf::process::stream_capture(), 746240116Smarcel static_cast< void * >(¶ms)); 747240116Smarcel 748240116Smarcel terminate_poll = false; 749240116Smarcel 750240116Smarcel const atf::tests::vars_map::const_iterator iter = metadata.find("timeout"); 751240116Smarcel INV(iter != metadata.end()); 752240116Smarcel const unsigned int timeout = 753240116Smarcel atf::text::to_type< unsigned int >((*iter).second); 754240116Smarcel const pid_t child_pid = child.pid(); 755240116Smarcel 756240116Smarcel // Get the input stream of stdout and stderr. 757240116Smarcel impl::file_handle outfh = child.stdout_fd(); 758240116Smarcel impl::file_handle errfh = child.stderr_fd(); 759240116Smarcel 760240116Smarcel bool timed_out = false; 761240116Smarcel 762240116Smarcel // Process the test case's output and multiplex it into our output 763240116Smarcel // stream as we read it. 764240116Smarcel int fds[2] = {outfh.get(), errfh.get()}; 765240116Smarcel child_muxer mux(fds, 2, writer); 766240116Smarcel try { 767240116Smarcel child_timer timeout_timer(timeout, child_pid, terminate_poll); 768240116Smarcel signal_programmer sigchld(SIGCHLD, sigchld_handler); 769240116Smarcel mux.mux(terminate_poll); 770240116Smarcel timed_out = timeout_timer.fired(); 771240116Smarcel } catch (...) { 772240116Smarcel UNREACHABLE; 773240116Smarcel } 774240116Smarcel 775240116Smarcel ::killpg(child_pid, SIGKILL); 776240116Smarcel mux.flush(); 777240116Smarcel atf::process::status status = child.wait(); 778240116Smarcel 779240116Smarcel std::string reason; 780240116Smarcel 781240116Smarcel if (timed_out) { 782240116Smarcel // Don't assume the child process has been signaled due to the timeout 783240116Smarcel // expiration as older versions did. The child process may have exited 784240116Smarcel // but we may have timed out due to a subchild process getting stuck. 785240116Smarcel reason = "Test case timed out after " + atf::text::to_string(timeout) + 786240116Smarcel " " + (timeout == 1 ? "second" : "seconds"); 787240116Smarcel } 788240116Smarcel 789240116Smarcel return std::make_pair(reason, status); 790240116Smarcel} 791