1// Copyright 2014 The Kyua Authors. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above copyright 11// notice, this list of conditions and the following disclaimer in the 12// documentation and/or other materials provided with the distribution. 13// * Neither the name of Google Inc. nor the names of its contributors 14// may be used to endorse or promote products derived from this software 15// without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29#include "engine/scanner.hpp" 30 31#include <cstdarg> 32#include <cstddef> 33#include <typeinfo> 34 35#include <atf-c++.hpp> 36 37#include "engine/filters.hpp" 38#include "model/metadata.hpp" 39#include "model/test_case.hpp" 40#include "model/test_program.hpp" 41#include "utils/format/containers.ipp" 42#include "utils/fs/path.hpp" 43#include "utils/optional.ipp" 44 45namespace fs = utils::fs; 46 47using utils::optional; 48 49 50namespace { 51 52 53/// Test program that implements a mock test_cases() lazy call. 54class mock_test_program : public model::test_program { 55 /// Number of times test_cases has been called. 56 mutable std::size_t _num_calls; 57 58 /// Collection of test cases; lazily initialized. 59 mutable model::test_cases_map _test_cases; 60 61public: 62 /// Constructs a new test program. 63 /// 64 /// \param binary_ The name of the test program binary relative to root_. 65 mock_test_program(const fs::path& binary_) : 66 test_program("unused-interface", binary_, fs::path("unused-root"), 67 "unused-suite", model::metadata_builder().build(), 68 model::test_cases_map()), 69 _num_calls(0) 70 { 71 } 72 73 /// Gets or loads the list of test cases from the test program. 74 /// 75 /// \return The list of test cases provided by the test program. 76 const model::test_cases_map& 77 test_cases(void) const 78 { 79 if (_num_calls == 0) { 80 const model::metadata metadata = model::metadata_builder().build(); 81 const model::test_case tc1("one", metadata); 82 const model::test_case tc2("two", metadata); 83 _test_cases.insert(model::test_cases_map::value_type("one", tc1)); 84 _test_cases.insert(model::test_cases_map::value_type("two", tc2)); 85 } 86 _num_calls++; 87 return _test_cases; 88 } 89 90 /// Returns the number of times test_cases() has been called. 91 /// 92 /// \return A counter. 93 std::size_t 94 num_calls(void) const 95 { 96 return _num_calls; 97 } 98}; 99 100 101/// Syntactic sugar to instantiate a test program with various test cases. 102/// 103/// The scanner only cares about the relative path of the test program object 104/// and the names of the test cases. This function helps in instantiating a 105/// test program that has the minimum set of details only. 106/// 107/// \param relative_path Relative path to the test program. 108/// \param ... List of test case names to add to the test program. Must be 109/// NULL-terminated. 110/// 111/// \return A constructed test program. 112static model::test_program_ptr 113new_test_program(const char* relative_path, ...) 114{ 115 model::test_program_builder builder( 116 "unused-interface", fs::path(relative_path), fs::path("unused-root"), 117 "unused-suite"); 118 119 va_list ap; 120 va_start(ap, relative_path); 121 const char* test_case_name; 122 while ((test_case_name = va_arg(ap, const char*)) != NULL) { 123 builder.add_test_case(test_case_name); 124 } 125 va_end(ap); 126 127 return builder.build_ptr(); 128} 129 130 131/// Yields all test cases in the scanner for simplicity of testing. 132/// 133/// In most of the tests below, we just care about the scanner returning the 134/// full set of matching test cases, not the specific behavior of every single 135/// yield() call. This function just returns the whole set, which helps in 136/// writing functional tests. 137/// 138/// \param scanner The scanner on which to iterate. 139/// 140/// \return The full collection of results yielded by the scanner. 141static std::set< engine::scan_result > 142yield_all(engine::scanner& scanner) 143{ 144 std::set< engine::scan_result > results; 145 while (!scanner.done()) { 146 const optional< engine::scan_result > result = scanner.yield(); 147 ATF_REQUIRE(result); 148 results.insert(result.get()); 149 } 150 ATF_REQUIRE(!scanner.yield()); 151 ATF_REQUIRE(scanner.done()); 152 return results; 153} 154 155 156} // anonymous namespace 157 158 159ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__no_tests); 160ATF_TEST_CASE_BODY(scanner__no_filters__no_tests) 161{ 162 const model::test_programs_vector test_programs; 163 const std::set< engine::test_filter > filters; 164 165 engine::scanner scanner(test_programs, filters); 166 ATF_REQUIRE(scanner.done()); 167 ATF_REQUIRE(!scanner.yield()); 168 ATF_REQUIRE(scanner.unused_filters().empty()); 169} 170 171 172ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__one_test_in_one_program); 173ATF_TEST_CASE_BODY(scanner__no_filters__one_test_in_one_program) 174{ 175 const model::test_program_ptr test_program = new_test_program( 176 "dir/program", "lone_test", NULL); 177 178 model::test_programs_vector test_programs; 179 test_programs.push_back(test_program); 180 181 const std::set< engine::test_filter > filters; 182 183 std::set< engine::scan_result > exp_results; 184 exp_results.insert(engine::scan_result(test_program, "lone_test")); 185 186 engine::scanner scanner(test_programs, filters); 187 const std::set< engine::scan_result > results = yield_all(scanner); 188 ATF_REQUIRE_EQ(exp_results, results); 189 ATF_REQUIRE(scanner.unused_filters().empty()); 190} 191 192 193ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__one_test_per_many_programs); 194ATF_TEST_CASE_BODY(scanner__no_filters__one_test_per_many_programs) 195{ 196 const model::test_program_ptr test_program1 = new_test_program( 197 "dir/program1", "foo_test", NULL); 198 const model::test_program_ptr test_program2 = new_test_program( 199 "program2", "bar_test", NULL); 200 const model::test_program_ptr test_program3 = new_test_program( 201 "a/b/c/d/e/program3", "baz_test", NULL); 202 203 model::test_programs_vector test_programs; 204 test_programs.push_back(test_program1); 205 test_programs.push_back(test_program2); 206 test_programs.push_back(test_program3); 207 208 const std::set< engine::test_filter > filters; 209 210 std::set< engine::scan_result > exp_results; 211 exp_results.insert(engine::scan_result(test_program1, "foo_test")); 212 exp_results.insert(engine::scan_result(test_program2, "bar_test")); 213 exp_results.insert(engine::scan_result(test_program3, "baz_test")); 214 215 engine::scanner scanner(test_programs, filters); 216 const std::set< engine::scan_result > results = yield_all(scanner); 217 ATF_REQUIRE_EQ(exp_results, results); 218 ATF_REQUIRE(scanner.unused_filters().empty()); 219} 220 221 222ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__many_tests_in_one_program); 223ATF_TEST_CASE_BODY(scanner__no_filters__many_tests_in_one_program) 224{ 225 const model::test_program_ptr test_program = new_test_program( 226 "dir/program", "first_test", "second_test", "third_test", NULL); 227 228 model::test_programs_vector test_programs; 229 test_programs.push_back(test_program); 230 231 const std::set< engine::test_filter > filters; 232 233 std::set< engine::scan_result > exp_results; 234 exp_results.insert(engine::scan_result(test_program, "first_test")); 235 exp_results.insert(engine::scan_result(test_program, "second_test")); 236 exp_results.insert(engine::scan_result(test_program, "third_test")); 237 238 engine::scanner scanner(test_programs, filters); 239 const std::set< engine::scan_result > results = yield_all(scanner); 240 ATF_REQUIRE_EQ(exp_results, results); 241 ATF_REQUIRE(scanner.unused_filters().empty()); 242} 243 244 245ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__many_tests_per_many_programs); 246ATF_TEST_CASE_BODY(scanner__no_filters__many_tests_per_many_programs) 247{ 248 const model::test_program_ptr test_program1 = new_test_program( 249 "dir/program1", "foo_test", "bar_test", "baz_test", NULL); 250 const model::test_program_ptr test_program2 = new_test_program( 251 "program2", "lone_test", NULL); 252 const model::test_program_ptr test_program3 = new_test_program( 253 "a/b/c/d/e/program3", "another_test", "last_test", NULL); 254 255 model::test_programs_vector test_programs; 256 test_programs.push_back(test_program1); 257 test_programs.push_back(test_program2); 258 test_programs.push_back(test_program3); 259 260 const std::set< engine::test_filter > filters; 261 262 std::set< engine::scan_result > exp_results; 263 exp_results.insert(engine::scan_result(test_program1, "foo_test")); 264 exp_results.insert(engine::scan_result(test_program1, "bar_test")); 265 exp_results.insert(engine::scan_result(test_program1, "baz_test")); 266 exp_results.insert(engine::scan_result(test_program2, "lone_test")); 267 exp_results.insert(engine::scan_result(test_program3, "another_test")); 268 exp_results.insert(engine::scan_result(test_program3, "last_test")); 269 270 engine::scanner scanner(test_programs, filters); 271 const std::set< engine::scan_result > results = yield_all(scanner); 272 ATF_REQUIRE_EQ(exp_results, results); 273 ATF_REQUIRE(scanner.unused_filters().empty()); 274} 275 276 277ATF_TEST_CASE_WITHOUT_HEAD(scanner__no_filters__verify_lazy_loads); 278ATF_TEST_CASE_BODY(scanner__no_filters__verify_lazy_loads) 279{ 280 const model::test_program_ptr test_program1(new mock_test_program( 281 fs::path("first"))); 282 const mock_test_program* mock_program1 = 283 dynamic_cast< const mock_test_program* >(test_program1.get()); 284 const model::test_program_ptr test_program2(new mock_test_program( 285 fs::path("second"))); 286 const mock_test_program* mock_program2 = 287 dynamic_cast< const mock_test_program* >(test_program2.get()); 288 289 model::test_programs_vector test_programs; 290 test_programs.push_back(test_program1); 291 test_programs.push_back(test_program2); 292 293 const std::set< engine::test_filter > filters; 294 295 std::set< engine::scan_result > exp_results; 296 exp_results.insert(engine::scan_result(test_program1, "one")); 297 exp_results.insert(engine::scan_result(test_program1, "two")); 298 exp_results.insert(engine::scan_result(test_program2, "one")); 299 exp_results.insert(engine::scan_result(test_program2, "two")); 300 301 engine::scanner scanner(test_programs, filters); 302 std::set< engine::scan_result > results; 303 ATF_REQUIRE_EQ(0, mock_program1->num_calls()); 304 ATF_REQUIRE_EQ(0, mock_program2->num_calls()); 305 306 // This abuses the internal implementation of the scanner by making 307 // assumptions on the order of the results. 308 results.insert(scanner.yield().get()); 309 ATF_REQUIRE_EQ(1, mock_program1->num_calls()); 310 ATF_REQUIRE_EQ(0, mock_program2->num_calls()); 311 results.insert(scanner.yield().get()); 312 ATF_REQUIRE_EQ(1, mock_program1->num_calls()); 313 ATF_REQUIRE_EQ(0, mock_program2->num_calls()); 314 results.insert(scanner.yield().get()); 315 ATF_REQUIRE_EQ(1, mock_program1->num_calls()); 316 ATF_REQUIRE_EQ(1, mock_program2->num_calls()); 317 results.insert(scanner.yield().get()); 318 ATF_REQUIRE_EQ(1, mock_program1->num_calls()); 319 ATF_REQUIRE_EQ(1, mock_program2->num_calls()); 320 ATF_REQUIRE(scanner.done()); 321 322 ATF_REQUIRE_EQ(exp_results, results); 323 ATF_REQUIRE(scanner.unused_filters().empty()); 324 325 // Make sure we are still talking to the original objects. 326 for (std::set< engine::scan_result >::const_iterator iter = results.begin(); 327 iter != results.end(); ++iter) { 328 const mock_test_program* mock_program = 329 dynamic_cast< const mock_test_program* >((*iter).first.get()); 330 ATF_REQUIRE_EQ(1, mock_program->num_calls()); 331 } 332} 333 334 335ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__no_tests); 336ATF_TEST_CASE_BODY(scanner__with_filters__no_tests) 337{ 338 const model::test_programs_vector test_programs; 339 340 std::set< engine::test_filter > filters; 341 filters.insert(engine::test_filter(fs::path("foo"), "bar")); 342 343 engine::scanner scanner(test_programs, filters); 344 ATF_REQUIRE(scanner.done()); 345 ATF_REQUIRE(!scanner.yield()); 346 ATF_REQUIRE_EQ(filters, scanner.unused_filters()); 347} 348 349 350ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__no_matches); 351ATF_TEST_CASE_BODY(scanner__with_filters__no_matches) 352{ 353 const model::test_program_ptr test_program1 = new_test_program( 354 "dir/program1", "foo_test", "bar_test", "baz_test", NULL); 355 const model::test_program_ptr test_program2 = new_test_program( 356 "dir/program2", "bar_test", NULL); 357 const model::test_program_ptr test_program3 = new_test_program( 358 "program3", "another_test", "last_test", NULL); 359 360 model::test_programs_vector test_programs; 361 test_programs.push_back(test_program1); 362 test_programs.push_back(test_program2); 363 test_programs.push_back(test_program3); 364 365 std::set< engine::test_filter > filters; 366 filters.insert(engine::test_filter(fs::path("dir/program2"), "baz_test")); 367 filters.insert(engine::test_filter(fs::path("program4"), "another_test")); 368 filters.insert(engine::test_filter(fs::path("dir/program3"), "")); 369 370 const std::set< engine::scan_result > exp_results; 371 372 engine::scanner scanner(test_programs, filters); 373 const std::set< engine::scan_result > results = yield_all(scanner); 374 ATF_REQUIRE_EQ(exp_results, results); 375 ATF_REQUIRE_EQ(filters, scanner.unused_filters()); 376} 377 378 379ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__some_matches); 380ATF_TEST_CASE_BODY(scanner__with_filters__some_matches) 381{ 382 const model::test_program_ptr test_program1 = new_test_program( 383 "dir/program1", "foo_test", "bar_test", "baz_test", NULL); 384 const model::test_program_ptr test_program2 = new_test_program( 385 "dir/program2", "bar_test", NULL); 386 const model::test_program_ptr test_program3 = new_test_program( 387 "program3", "another_test", "last_test", NULL); 388 const model::test_program_ptr test_program4 = new_test_program( 389 "program4", "more_test", NULL); 390 391 model::test_programs_vector test_programs; 392 test_programs.push_back(test_program1); 393 test_programs.push_back(test_program2); 394 test_programs.push_back(test_program3); 395 test_programs.push_back(test_program4); 396 397 std::set< engine::test_filter > filters; 398 filters.insert(engine::test_filter(fs::path("dir/program1"), "baz_test")); 399 filters.insert(engine::test_filter(fs::path("dir/program2"), "foo_test")); 400 filters.insert(engine::test_filter(fs::path("program3"), "")); 401 402 std::set< engine::test_filter > exp_filters; 403 exp_filters.insert(engine::test_filter(fs::path("dir/program2"), 404 "foo_test")); 405 406 std::set< engine::scan_result > exp_results; 407 exp_results.insert(engine::scan_result(test_program1, "baz_test")); 408 exp_results.insert(engine::scan_result(test_program3, "another_test")); 409 exp_results.insert(engine::scan_result(test_program3, "last_test")); 410 411 engine::scanner scanner(test_programs, filters); 412 const std::set< engine::scan_result > results = yield_all(scanner); 413 ATF_REQUIRE_EQ(exp_results, results); 414 415 ATF_REQUIRE_EQ(exp_filters, scanner.unused_filters()); 416} 417 418 419ATF_TEST_CASE_WITHOUT_HEAD(scanner__with_filters__verify_lazy_loads); 420ATF_TEST_CASE_BODY(scanner__with_filters__verify_lazy_loads) 421{ 422 const model::test_program_ptr test_program1(new mock_test_program( 423 fs::path("first"))); 424 const mock_test_program* mock_program1 = 425 dynamic_cast< const mock_test_program* >(test_program1.get()); 426 const model::test_program_ptr test_program2(new mock_test_program( 427 fs::path("second"))); 428 const mock_test_program* mock_program2 = 429 dynamic_cast< const mock_test_program* >(test_program2.get()); 430 431 model::test_programs_vector test_programs; 432 test_programs.push_back(test_program1); 433 test_programs.push_back(test_program2); 434 435 std::set< engine::test_filter > filters; 436 filters.insert(engine::test_filter(fs::path("first"), "")); 437 438 std::set< engine::scan_result > exp_results; 439 exp_results.insert(engine::scan_result(test_program1, "one")); 440 exp_results.insert(engine::scan_result(test_program1, "two")); 441 442 engine::scanner scanner(test_programs, filters); 443 std::set< engine::scan_result > results; 444 ATF_REQUIRE_EQ(0, mock_program1->num_calls()); 445 ATF_REQUIRE_EQ(0, mock_program2->num_calls()); 446 447 results.insert(scanner.yield().get()); 448 ATF_REQUIRE_EQ(1, mock_program1->num_calls()); 449 ATF_REQUIRE_EQ(0, mock_program2->num_calls()); 450 results.insert(scanner.yield().get()); 451 ATF_REQUIRE_EQ(1, mock_program1->num_calls()); 452 ATF_REQUIRE_EQ(0, mock_program2->num_calls()); 453 ATF_REQUIRE(scanner.done()); 454 455 ATF_REQUIRE_EQ(exp_results, results); 456 ATF_REQUIRE(scanner.unused_filters().empty()); 457 458 ATF_REQUIRE_EQ(1, mock_program1->num_calls()); 459 ATF_REQUIRE_EQ(0, mock_program2->num_calls()); 460} 461 462 463ATF_INIT_TEST_CASES(tcs) 464{ 465 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__no_tests); 466 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__one_test_in_one_program); 467 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__one_test_per_many_programs); 468 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__many_tests_in_one_program); 469 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__many_tests_per_many_programs); 470 ATF_ADD_TEST_CASE(tcs, scanner__no_filters__verify_lazy_loads); 471 472 ATF_ADD_TEST_CASE(tcs, scanner__with_filters__no_tests); 473 ATF_ADD_TEST_CASE(tcs, scanner__with_filters__no_matches); 474 ATF_ADD_TEST_CASE(tcs, scanner__with_filters__some_matches); 475 ATF_ADD_TEST_CASE(tcs, scanner__with_filters__verify_lazy_loads); 476} 477