1// Copyright 2012 The Kyua Authors. 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are 6// met: 7// 8// * Redistributions of source code must retain the above copyright 9// notice, this list of conditions and the following disclaimer. 10// * Redistributions in binary form must reproduce the above copyright 11// notice, this list of conditions and the following disclaimer in the 12// documentation and/or other materials provided with the distribution. 13// * Neither the name of Google Inc. nor the names of its contributors 14// may be used to endorse or promote products derived from this software 15// without specific prior written permission. 16// 17// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29#include "cli/cmd_report_html.hpp" 30 31#include <algorithm> 32#include <cerrno> 33#include <cstdlib> 34#include <set> 35#include <stdexcept> 36 37#include "cli/common.ipp" 38#include "drivers/scan_results.hpp" 39#include "engine/filters.hpp" 40#include "model/context.hpp" 41#include "model/metadata.hpp" 42#include "model/test_case.hpp" 43#include "model/test_program.hpp" 44#include "model/test_result.hpp" 45#include "store/layout.hpp" 46#include "store/read_transaction.hpp" 47#include "utils/cmdline/options.hpp" 48#include "utils/cmdline/parser.ipp" 49#include "utils/cmdline/ui.hpp" 50#include "utils/datetime.hpp" 51#include "utils/env.hpp" 52#include "utils/format/macros.hpp" 53#include "utils/fs/exceptions.hpp" 54#include "utils/fs/operations.hpp" 55#include "utils/fs/path.hpp" 56#include "utils/optional.ipp" 57#include "utils/text/operations.hpp" 58#include "utils/text/templates.hpp" 59 60namespace cmdline = utils::cmdline; 61namespace config = utils::config; 62namespace datetime = utils::datetime; 63namespace fs = utils::fs; 64namespace layout = store::layout; 65namespace text = utils::text; 66 67using utils::optional; 68 69 70namespace { 71 72 73/// Creates the report's top directory and fails if it exists. 74/// 75/// \param directory The directory to create. 76/// \param force Whether to wipe an existing directory or not. 77/// 78/// \throw std::runtime_error If the directory already exists; this is a user 79/// error that the user must correct. 80/// \throw fs::error If the directory creation fails for any other reason. 81static void 82create_top_directory(const fs::path& directory, const bool force) 83{ 84 if (force) { 85 if (fs::exists(directory)) 86 fs::rm_r(directory); 87 } 88 89 try { 90 fs::mkdir(directory, 0755); 91 } catch (const fs::system_error& e) { 92 if (e.original_errno() == EEXIST) 93 throw std::runtime_error(F("Output directory '%s' already exists; " 94 "maybe use --force?") % 95 directory); 96 else 97 throw e; 98 } 99} 100 101 102/// Generates a flat unique filename for a given test case. 103/// 104/// \param test_program The test program for which to genereate the name. 105/// \param test_case_name The test case name. 106/// 107/// \return A filename unique within a directory with a trailing HTML extension. 108static std::string 109test_case_filename(const model::test_program& test_program, 110 const std::string& test_case_name) 111{ 112 static const char* special_characters = "/:"; 113 114 std::string name = cli::format_test_case_id(test_program, test_case_name); 115 std::string::size_type pos = name.find_first_of(special_characters); 116 while (pos != std::string::npos) { 117 name.replace(pos, 1, "_"); 118 pos = name.find_first_of(special_characters, pos + 1); 119 } 120 return name + ".html"; 121} 122 123 124/// Adds a string to string map to the templates. 125/// 126/// \param [in,out] templates The templates to add the map to. 127/// \param props The map to add to the templates. 128/// \param key_vector Name of the template vector that holds the keys. 129/// \param value_vector Name of the template vector that holds the values. 130static void 131add_map(text::templates_def& templates, const config::properties_map& props, 132 const std::string& key_vector, const std::string& value_vector) 133{ 134 templates.add_vector(key_vector); 135 templates.add_vector(value_vector); 136 137 for (config::properties_map::const_iterator iter = props.begin(); 138 iter != props.end(); ++iter) { 139 templates.add_to_vector(key_vector, (*iter).first); 140 templates.add_to_vector(value_vector, (*iter).second); 141 } 142} 143 144 145/// Generates an HTML report. 146class html_hooks : public drivers::scan_results::base_hooks { 147 /// User interface object where to report progress. 148 cmdline::ui* _ui; 149 150 /// The top directory in which to create the HTML files. 151 fs::path _directory; 152 153 /// Collection of result types to include in the report. 154 const cli::result_types& _results_filters; 155 156 /// The start time of the first test. 157 optional< utils::datetime::timestamp > _start_time; 158 159 /// The end time of the last test. 160 optional< utils::datetime::timestamp > _end_time; 161 162 /// The total run time of the tests. Note that we cannot subtract _end_time 163 /// from _start_time to compute this due to parallel execution. 164 utils::datetime::delta _runtime; 165 166 /// Templates accumulator to generate the index.html file. 167 text::templates_def _summary_templates; 168 169 /// Mapping of result types to the amount of tests with such result. 170 std::map< model::test_result_type, std::size_t > _types_count; 171 172 /// Generates a common set of templates for all of our files. 173 /// 174 /// \return A new templates object with common parameters. 175 static text::templates_def 176 common_templates(void) 177 { 178 text::templates_def templates; 179 templates.add_variable("css", "report.css"); 180 return templates; 181 } 182 183 /// Adds a test case result to the summary. 184 /// 185 /// \param test_program The test program with the test case to be added. 186 /// \param test_case_name Name of the test case. 187 /// \param result The result of the test case. 188 /// \param has_detail If true, the result of the test case has not been 189 /// filtered and therefore there exists a separate file for the test 190 /// with all of its information. 191 void 192 add_to_summary(const model::test_program& test_program, 193 const std::string& test_case_name, 194 const model::test_result& result, 195 const bool has_detail) 196 { 197 ++_types_count[result.type()]; 198 199 if (!has_detail) 200 return; 201 202 std::string test_cases_vector; 203 std::string test_cases_file_vector; 204 switch (result.type()) { 205 case model::test_result_broken: 206 test_cases_vector = "broken_test_cases"; 207 test_cases_file_vector = "broken_test_cases_file"; 208 break; 209 210 case model::test_result_expected_failure: 211 test_cases_vector = "xfail_test_cases"; 212 test_cases_file_vector = "xfail_test_cases_file"; 213 break; 214 215 case model::test_result_failed: 216 test_cases_vector = "failed_test_cases"; 217 test_cases_file_vector = "failed_test_cases_file"; 218 break; 219 220 case model::test_result_passed: 221 test_cases_vector = "passed_test_cases"; 222 test_cases_file_vector = "passed_test_cases_file"; 223 break; 224 225 case model::test_result_skipped: 226 test_cases_vector = "skipped_test_cases"; 227 test_cases_file_vector = "skipped_test_cases_file"; 228 break; 229 } 230 INV(!test_cases_vector.empty()); 231 INV(!test_cases_file_vector.empty()); 232 233 _summary_templates.add_to_vector( 234 test_cases_vector, 235 cli::format_test_case_id(test_program, test_case_name)); 236 _summary_templates.add_to_vector( 237 test_cases_file_vector, 238 test_case_filename(test_program, test_case_name)); 239 } 240 241 /// Instantiate a template to generate an HTML file in the output directory. 242 /// 243 /// \param templates The templates to use. 244 /// \param template_name The name of the template. This is automatically 245 /// searched for in the installed directory, so do not provide a path. 246 /// \param output_name The name of the output file. This is a basename to 247 /// be created within the output directory. 248 /// 249 /// \throw text::error If there is any problem applying the templates. 250 void 251 generate(const text::templates_def& templates, 252 const std::string& template_name, 253 const std::string& output_name) const 254 { 255 const fs::path miscdir(utils::getenv_with_default( 256 "KYUA_MISCDIR", KYUA_MISCDIR)); 257 const fs::path template_file = miscdir / template_name; 258 const fs::path output_path(_directory / output_name); 259 260 _ui->out(F("Generating %s") % output_path); 261 text::instantiate(templates, template_file, output_path); 262 } 263 264 /// Gets the number of tests with a given result type. 265 /// 266 /// \param type The type to be queried. 267 /// 268 /// \return The number of tests of the given type, or 0 if none have yet 269 /// been registered by add_to_summary(). 270 std::size_t 271 get_count(const model::test_result_type type) const 272 { 273 const std::map< model::test_result_type, std::size_t >::const_iterator 274 iter = _types_count.find(type); 275 if (iter == _types_count.end()) 276 return 0; 277 else 278 return (*iter).second; 279 } 280 281public: 282 /// Constructor for the hooks. 283 /// 284 /// \param ui_ User interface object where to report progress. 285 /// \param directory_ The directory in which to create the HTML files. 286 /// \param results_filters_ The result types to include in the report. 287 /// Cannot be empty. 288 html_hooks(cmdline::ui* ui_, const fs::path& directory_, 289 const cli::result_types& results_filters_) : 290 _ui(ui_), 291 _directory(directory_), 292 _results_filters(results_filters_), 293 _summary_templates(common_templates()) 294 { 295 PRE(!results_filters_.empty()); 296 297 // Keep in sync with add_to_summary(). 298 _summary_templates.add_vector("broken_test_cases"); 299 _summary_templates.add_vector("broken_test_cases_file"); 300 _summary_templates.add_vector("xfail_test_cases"); 301 _summary_templates.add_vector("xfail_test_cases_file"); 302 _summary_templates.add_vector("failed_test_cases"); 303 _summary_templates.add_vector("failed_test_cases_file"); 304 _summary_templates.add_vector("passed_test_cases"); 305 _summary_templates.add_vector("passed_test_cases_file"); 306 _summary_templates.add_vector("skipped_test_cases"); 307 _summary_templates.add_vector("skipped_test_cases_file"); 308 } 309 310 /// Callback executed when the context is loaded. 311 /// 312 /// \param context The context loaded from the database. 313 void 314 got_context(const model::context& context) 315 { 316 text::templates_def templates = common_templates(); 317 templates.add_variable("cwd", context.cwd().str()); 318 add_map(templates, context.env(), "env_var", "env_var_value"); 319 generate(templates, "context.html", "context.html"); 320 } 321 322 /// Callback executed when a test results is found. 323 /// 324 /// \param iter Container for the test result's data. 325 void 326 got_result(store::results_iterator& iter) 327 { 328 const model::test_program_ptr test_program = iter.test_program(); 329 const std::string& test_case_name = iter.test_case_name(); 330 const model::test_result result = iter.result(); 331 332 if (std::find(_results_filters.begin(), _results_filters.end(), 333 result.type()) == _results_filters.end()) { 334 add_to_summary(*test_program, test_case_name, result, false); 335 return; 336 } 337 338 add_to_summary(*test_program, test_case_name, result, true); 339 340 if (!_start_time || _start_time.get() > iter.start_time()) 341 _start_time = iter.start_time(); 342 if (!_end_time || _end_time.get() < iter.end_time()) 343 _end_time = iter.end_time(); 344 345 const datetime::delta duration = iter.end_time() - iter.start_time(); 346 347 _runtime += duration; 348 349 text::templates_def templates = common_templates(); 350 templates.add_variable("test_case", 351 cli::format_test_case_id(*test_program, 352 test_case_name)); 353 templates.add_variable("test_program", 354 test_program->absolute_path().str()); 355 templates.add_variable("result", cli::format_result(result)); 356 templates.add_variable("start_time", 357 iter.start_time().to_iso8601_in_utc()); 358 templates.add_variable("end_time", 359 iter.end_time().to_iso8601_in_utc()); 360 templates.add_variable("duration", cli::format_delta(duration)); 361 362 const model::test_case& test_case = test_program->find(test_case_name); 363 add_map(templates, test_case.get_metadata().to_properties(), 364 "metadata_var", "metadata_value"); 365 366 { 367 const std::string stdout_text = iter.stdout_contents(); 368 if (!stdout_text.empty()) 369 templates.add_variable("stdout", text::escape_xml(stdout_text)); 370 } 371 { 372 const std::string stderr_text = iter.stderr_contents(); 373 if (!stderr_text.empty()) 374 templates.add_variable("stderr", text::escape_xml(stderr_text)); 375 } 376 377 generate(templates, "test_result.html", 378 test_case_filename(*test_program, test_case_name)); 379 } 380 381 /// Writes the index.html file in the output directory. 382 /// 383 /// This should only be called once all the processing has been done; 384 /// i.e. when the scan_results driver returns. 385 void 386 write_summary(void) 387 { 388 const std::size_t n_passed = get_count(model::test_result_passed); 389 const std::size_t n_failed = get_count(model::test_result_failed); 390 const std::size_t n_skipped = get_count(model::test_result_skipped); 391 const std::size_t n_xfail = get_count( 392 model::test_result_expected_failure); 393 const std::size_t n_broken = get_count(model::test_result_broken); 394 395 const std::size_t n_bad = n_broken + n_failed; 396 397 if (_start_time) { 398 INV(_end_time); 399 _summary_templates.add_variable( 400 "start_time", _start_time.get().to_iso8601_in_utc()); 401 _summary_templates.add_variable( 402 "end_time", _end_time.get().to_iso8601_in_utc()); 403 } else { 404 _summary_templates.add_variable("start_time", "No tests run"); 405 _summary_templates.add_variable("end_time", "No tests run"); 406 } 407 _summary_templates.add_variable("duration", 408 cli::format_delta(_runtime)); 409 _summary_templates.add_variable("passed_tests_count", 410 F("%s") % n_passed); 411 _summary_templates.add_variable("failed_tests_count", 412 F("%s") % n_failed); 413 _summary_templates.add_variable("skipped_tests_count", 414 F("%s") % n_skipped); 415 _summary_templates.add_variable("xfail_tests_count", 416 F("%s") % n_xfail); 417 _summary_templates.add_variable("broken_tests_count", 418 F("%s") % n_broken); 419 _summary_templates.add_variable("bad_tests_count", F("%s") % n_bad); 420 421 generate(text::templates_def(), "report.css", "report.css"); 422 generate(_summary_templates, "index.html", "index.html"); 423 } 424}; 425 426 427} // anonymous namespace 428 429 430/// Default constructor for cmd_report_html. 431cli::cmd_report_html::cmd_report_html(void) : cli_command( 432 "report-html", "", 0, 0, 433 "Generates an HTML report with the result of a test suite run") 434{ 435 add_option(results_file_open_option); 436 add_option(cmdline::bool_option( 437 "force", "Wipe the output directory before generating the new report; " 438 "use care")); 439 add_option(cmdline::path_option( 440 "output", "The directory in which to store the HTML files", 441 "path", "html")); 442 add_option(cmdline::list_option( 443 "results-filter", "Comma-separated list of result types to include in " 444 "the report", "types", "skipped,xfail,broken,failed")); 445} 446 447 448/// Entry point for the "report-html" subcommand. 449/// 450/// \param ui Object to interact with the I/O of the program. 451/// \param cmdline Representation of the command line to the subcommand. 452/// 453/// \return 0 if everything is OK, 1 if the statement is invalid or if there is 454/// any other problem. 455int 456cli::cmd_report_html::run(cmdline::ui* ui, 457 const cmdline::parsed_cmdline& cmdline, 458 const config::tree& /* user_config */) 459{ 460 const result_types types = get_result_types(cmdline); 461 462 const fs::path results_file = layout::find_results( 463 results_file_open(cmdline)); 464 465 const fs::path directory = 466 cmdline.get_option< cmdline::path_option >("output"); 467 create_top_directory(directory, cmdline.has_option("force")); 468 html_hooks hooks(ui, directory, types); 469 drivers::scan_results::drive(results_file, 470 std::set< engine::test_filter >(), 471 hooks); 472 hooks.write_summary(); 473 474 return EXIT_SUCCESS; 475} 476