1// Copyright 2011 The Kyua Authors. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above copyright 11// notice, this list of conditions and the following disclaimer in the 12// documentation and/or other materials provided with the distribution. 13// * Neither the name of Google Inc. nor the names of its contributors 14// may be used to endorse or promote products derived from this software 15// without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29#include "drivers/list_tests.hpp" 30 31extern "C" { 32#include <sys/stat.h> 33 34#include <unistd.h> 35} 36 37#include <map> 38#include <set> 39#include <string> 40 41#include <atf-c++.hpp> 42 43#include "cli/cmd_list.hpp" 44#include "cli/common.ipp" 45#include "engine/atf.hpp" 46#include "engine/config.hpp" 47#include "engine/exceptions.hpp" 48#include "engine/filters.hpp" 49#include "engine/scheduler.hpp" 50#include "model/metadata.hpp" 51#include "model/test_case.hpp" 52#include "model/test_program.hpp" 53#include "utils/config/tree.ipp" 54#include "utils/env.hpp" 55#include "utils/format/macros.hpp" 56#include "utils/optional.ipp" 57#include "utils/test_utils.ipp" 58 59namespace config = utils::config; 60namespace fs = utils::fs; 61namespace scheduler = engine::scheduler; 62 63using utils::none; 64using utils::optional; 65 66 67namespace { 68 69 70/// Gets the path to the helpers for this test program. 71/// 72/// \param test_case A pointer to the currently running test case. 73/// 74/// \return The path to the helpers binary. 75static fs::path 76helpers(const atf::tests::tc* test_case) 77{ 78 return fs::path(test_case->get_config_var("srcdir")) / 79 "list_tests_helpers"; 80} 81 82 83/// Hooks to capture the incremental listing of test cases. 84class capture_hooks : public drivers::list_tests::base_hooks { 85public: 86 /// Set of the listed test cases in a program:test_case form. 87 std::set< std::string > test_cases; 88 89 /// Set of the listed test cases in a program:test_case form. 90 std::map< std::string, model::metadata > metadatas; 91 92 /// Called when a test case is identified in a test suite. 93 /// 94 /// \param test_program The test program containing the test case. 95 /// \param test_case_name The name of the located test case. 96 virtual void 97 got_test_case(const model::test_program& test_program, 98 const std::string& test_case_name) 99 { 100 const std::string ident = F("%s:%s") % 101 test_program.relative_path() % test_case_name; 102 test_cases.insert(ident); 103 104 metadatas.insert(std::map< std::string, model::metadata >::value_type( 105 ident, test_program.find(test_case_name).get_metadata())); 106 } 107}; 108 109 110/// Creates a mock test suite. 111/// 112/// \param tc Pointer to the caller test case; needed to obtain the srcdir 113/// variable of the caller. 114/// \param source_root Basename of the directory that will contain the 115/// Kyuafiles. 116/// \param build_root Basename of the directory that will contain the test 117/// programs. May or may not be the same as source_root. 118static void 119create_helpers(const atf::tests::tc* tc, const fs::path& source_root, 120 const fs::path& build_root) 121{ 122 ATF_REQUIRE(::mkdir(source_root.c_str(), 0755) != -1); 123 ATF_REQUIRE(::mkdir((source_root / "dir").c_str(), 0755) != -1); 124 if (source_root != build_root) { 125 ATF_REQUIRE(::mkdir(build_root.c_str(), 0755) != -1); 126 ATF_REQUIRE(::mkdir((build_root / "dir").c_str(), 0755) != -1); 127 } 128 ATF_REQUIRE(::symlink(helpers(tc).c_str(), 129 (build_root / "dir/program").c_str()) != -1); 130 131 atf::utils::create_file( 132 (source_root / "Kyuafile").str(), 133 "syntax(2)\n" 134 "include('dir/Kyuafile')\n"); 135 136 atf::utils::create_file( 137 (source_root / "dir/Kyuafile").str(), 138 "syntax(2)\n" 139 "atf_test_program{name='program', test_suite='suite-name'}\n"); 140} 141 142 143/// Runs the mock test suite. 144/// 145/// \param source_root Path to the directory that contains the Kyuafiles. 146/// \param build_root If not none, path to the directory that contains the test 147/// programs. 148/// \param hooks The hooks to use during the listing. 149/// \param filter_program If not null, the filter on the test program name. 150/// \param filter_test_case If not null, the filter on the test case name. 151/// \param the_variable If not null, the value to pass to the test program as 152/// its "the-variable" configuration property. 153/// 154/// \return The result data of the driver. 155static drivers::list_tests::result 156run_helpers(const fs::path& source_root, 157 const optional< fs::path > build_root, 158 drivers::list_tests::base_hooks& hooks, 159 const char* filter_program = NULL, 160 const char* filter_test_case = NULL, 161 const char* the_variable = NULL) 162{ 163 std::set< engine::test_filter > filters; 164 if (filter_program != NULL && filter_test_case != NULL) 165 filters.insert(engine::test_filter(fs::path(filter_program), 166 filter_test_case)); 167 168 config::tree user_config = engine::empty_config(); 169 if (the_variable != NULL) { 170 user_config.set_string("test_suites.suite-name.the-variable", 171 the_variable); 172 } 173 174 return drivers::list_tests::drive(source_root / "Kyuafile", build_root, 175 filters, user_config, hooks); 176} 177 178 179} // anonymous namespace 180 181 182ATF_TEST_CASE_WITHOUT_HEAD(one_test_case); 183ATF_TEST_CASE_BODY(one_test_case) 184{ 185 utils::setenv("TESTS", "some_properties"); 186 capture_hooks hooks; 187 create_helpers(this, fs::path("root"), fs::path("root")); 188 run_helpers(fs::path("root"), none, hooks); 189 190 std::set< std::string > exp_test_cases; 191 exp_test_cases.insert("dir/program:some_properties"); 192 ATF_REQUIRE(exp_test_cases == hooks.test_cases); 193} 194 195 196ATF_TEST_CASE_WITHOUT_HEAD(many_test_cases); 197ATF_TEST_CASE_BODY(many_test_cases) 198{ 199 utils::setenv("TESTS", "no_properties some_properties"); 200 capture_hooks hooks; 201 create_helpers(this, fs::path("root"), fs::path("root")); 202 run_helpers(fs::path("root"), none, hooks); 203 204 std::set< std::string > exp_test_cases; 205 exp_test_cases.insert("dir/program:no_properties"); 206 exp_test_cases.insert("dir/program:some_properties"); 207 ATF_REQUIRE(exp_test_cases == hooks.test_cases); 208} 209 210 211ATF_TEST_CASE_WITHOUT_HEAD(filter_match); 212ATF_TEST_CASE_BODY(filter_match) 213{ 214 utils::setenv("TESTS", "no_properties some_properties"); 215 capture_hooks hooks; 216 create_helpers(this, fs::path("root"), fs::path("root")); 217 run_helpers(fs::path("root"), none, hooks, "dir/program", 218 "some_properties"); 219 220 std::set< std::string > exp_test_cases; 221 exp_test_cases.insert("dir/program:some_properties"); 222 ATF_REQUIRE(exp_test_cases == hooks.test_cases); 223} 224 225 226ATF_TEST_CASE_WITHOUT_HEAD(build_root); 227ATF_TEST_CASE_BODY(build_root) 228{ 229 utils::setenv("TESTS", "no_properties some_properties"); 230 capture_hooks hooks; 231 create_helpers(this, fs::path("source"), fs::path("build")); 232 run_helpers(fs::path("source"), utils::make_optional(fs::path("build")), 233 hooks); 234 235 std::set< std::string > exp_test_cases; 236 exp_test_cases.insert("dir/program:no_properties"); 237 exp_test_cases.insert("dir/program:some_properties"); 238 ATF_REQUIRE(exp_test_cases == hooks.test_cases); 239} 240 241 242ATF_TEST_CASE_WITHOUT_HEAD(config_in_head); 243ATF_TEST_CASE_BODY(config_in_head) 244{ 245 utils::setenv("TESTS", "config_in_head"); 246 capture_hooks hooks; 247 create_helpers(this, fs::path("source"), fs::path("build")); 248 run_helpers(fs::path("source"), utils::make_optional(fs::path("build")), 249 hooks, NULL, NULL, "magic value"); 250 251 std::set< std::string > exp_test_cases; 252 exp_test_cases.insert("dir/program:config_in_head"); 253 ATF_REQUIRE(exp_test_cases == hooks.test_cases); 254 255 const model::metadata& metadata = hooks.metadatas.find( 256 "dir/program:config_in_head")->second; 257 ATF_REQUIRE_EQ("the-variable is magic value", metadata.description()); 258} 259 260 261ATF_TEST_CASE_WITHOUT_HEAD(crash); 262ATF_TEST_CASE_BODY(crash) 263{ 264 utils::setenv("TESTS", "crash_list some_properties"); 265 capture_hooks hooks; 266 create_helpers(this, fs::path("root"), fs::path("root")); 267 run_helpers(fs::path("root"), none, hooks, "dir/program"); 268 269 std::set< std::string > exp_test_cases; 270 exp_test_cases.insert("dir/program:__test_cases_list__"); 271 ATF_REQUIRE(exp_test_cases == hooks.test_cases); 272} 273 274 275ATF_INIT_TEST_CASES(tcs) 276{ 277 scheduler::register_interface( 278 "atf", std::shared_ptr< scheduler::interface >( 279 new engine::atf_interface())); 280 281 ATF_ADD_TEST_CASE(tcs, one_test_case); 282 ATF_ADD_TEST_CASE(tcs, many_test_cases); 283 ATF_ADD_TEST_CASE(tcs, filter_match); 284 ATF_ADD_TEST_CASE(tcs, build_root); 285 ATF_ADD_TEST_CASE(tcs, config_in_head); 286 ATF_ADD_TEST_CASE(tcs, crash); 287} 288