check_test.cpp revision 260029 
1//
2// Automated Testing Framework (atf)
3//
4// Copyright (c) 2007 The NetBSD Foundation, Inc.
5// All rights reserved.
6//
7// Redistribution and use in source and binary forms, with or without
8// modification, are permitted provided that the following conditions
9// are met:
10// 1. Redistributions of source code must retain the above copyright
11//    notice, this list of conditions and the following disclaimer.
12// 2. Redistributions in binary form must reproduce the above copyright
13//    notice, this list of conditions and the following disclaimer in the
14//    documentation and/or other materials provided with the distribution.
15//
16// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
17// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
18// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
19// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
21// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
23// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
25// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
26// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
27// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28//
29
30extern "C" {
31#include <fcntl.h>
32#include <signal.h>
33#include <unistd.h>
34}
35
36#include <cstdlib>
37#include <cstring>
38#include <fstream>
39#include <iostream>
40#include <list>
41#include <memory>
42#include <vector>
43
44#include <atf-c++.hpp>
45
46#include "check.hpp"
47#include "config.hpp"
48#include "utils.hpp"
49
50#include "detail/fs.hpp"
51#include "detail/process.hpp"
52#include "detail/test_helpers.hpp"
53#include "detail/text.hpp"
54
55// ------------------------------------------------------------------------
56// Auxiliary functions.
57// ------------------------------------------------------------------------
58
59static
60std::auto_ptr< atf::check::check_result >
61do_exec(const atf::tests::tc* tc, const char* helper_name)
62{
63    std::vector< std::string > argv;
64    argv.push_back(get_process_helpers_path(*tc, false).str());
65    argv.push_back(helper_name);
66    std::cout << "Executing " << argv[0] << " " << argv[1] << "\n";
67
68    atf::process::argv_array argva(argv);
69    return atf::check::exec(argva);
70}
71
72static
73std::auto_ptr< atf::check::check_result >
74do_exec(const atf::tests::tc* tc, const char* helper_name, const char *carg2)
75{
76    std::vector< std::string > argv;
77    argv.push_back(get_process_helpers_path(*tc, false).str());
78    argv.push_back(helper_name);
79    argv.push_back(carg2);
80    std::cout << "Executing " << argv[0] << " " << argv[1] << " "
81              << argv[2] << "\n";
82
83    atf::process::argv_array argva(argv);
84    return atf::check::exec(argva);
85}
86
87// ------------------------------------------------------------------------
88// Helper test cases for the free functions.
89// ------------------------------------------------------------------------
90
91ATF_TEST_CASE(h_build_c_o_ok);
92ATF_TEST_CASE_HEAD(h_build_c_o_ok)
93{
94    set_md_var("descr", "Helper test case for build_c_o");
95}
96ATF_TEST_CASE_BODY(h_build_c_o_ok)
97{
98    std::ofstream sfile("test.c");
99    sfile << "#include <stdio.h>\n";
100    sfile.close();
101
102    ATF_REQUIRE(atf::check::build_c_o("test.c", "test.o",
103                                      atf::process::argv_array()));
104}
105
106ATF_TEST_CASE(h_build_c_o_fail);
107ATF_TEST_CASE_HEAD(h_build_c_o_fail)
108{
109    set_md_var("descr", "Helper test case for build_c_o");
110}
111ATF_TEST_CASE_BODY(h_build_c_o_fail)
112{
113    std::ofstream sfile("test.c");
114    sfile << "void foo(void) { int a = UNDEFINED_SYMBOL; }\n";
115    sfile.close();
116
117    ATF_REQUIRE(!atf::check::build_c_o("test.c", "test.o",
118                                       atf::process::argv_array()));
119}
120
121ATF_TEST_CASE(h_build_cpp_ok);
122ATF_TEST_CASE_HEAD(h_build_cpp_ok)
123{
124    set_md_var("descr", "Helper test case for build_cpp");
125}
126ATF_TEST_CASE_BODY(h_build_cpp_ok)
127{
128    std::ofstream sfile("test.c");
129    sfile << "#define A foo\n";
130    sfile << "#define B bar\n";
131    sfile << "A B\n";
132    sfile.close();
133
134    ATF_REQUIRE(atf::check::build_cpp("test.c", "test.p",
135                                      atf::process::argv_array()));
136}
137
138ATF_TEST_CASE(h_build_cpp_fail);
139ATF_TEST_CASE_HEAD(h_build_cpp_fail)
140{
141    set_md_var("descr", "Helper test case for build_cpp");
142}
143ATF_TEST_CASE_BODY(h_build_cpp_fail)
144{
145    std::ofstream sfile("test.c");
146    sfile << "#include \"./non-existent.h\"\n";
147    sfile.close();
148
149    ATF_REQUIRE(!atf::check::build_cpp("test.c", "test.p",
150                                       atf::process::argv_array()));
151}
152
153ATF_TEST_CASE(h_build_cxx_o_ok);
154ATF_TEST_CASE_HEAD(h_build_cxx_o_ok)
155{
156    set_md_var("descr", "Helper test case for build_cxx_o");
157}
158ATF_TEST_CASE_BODY(h_build_cxx_o_ok)
159{
160    std::ofstream sfile("test.cpp");
161    sfile << "#include <iostream>\n";
162    sfile.close();
163
164    ATF_REQUIRE(atf::check::build_cxx_o("test.cpp", "test.o",
165                                        atf::process::argv_array()));
166}
167
168ATF_TEST_CASE(h_build_cxx_o_fail);
169ATF_TEST_CASE_HEAD(h_build_cxx_o_fail)
170{
171    set_md_var("descr", "Helper test case for build_cxx_o");
172}
173ATF_TEST_CASE_BODY(h_build_cxx_o_fail)
174{
175    std::ofstream sfile("test.cpp");
176    sfile << "void foo(void) { int a = UNDEFINED_SYMBOL; }\n";
177    sfile.close();
178
179    ATF_REQUIRE(!atf::check::build_cxx_o("test.cpp", "test.o",
180                                         atf::process::argv_array()));
181}
182
183// ------------------------------------------------------------------------
184// Test cases for the free functions.
185// ------------------------------------------------------------------------
186
187ATF_TEST_CASE(build_c_o);
188ATF_TEST_CASE_HEAD(build_c_o)
189{
190    set_md_var("descr", "Tests the build_c_o function");
191}
192ATF_TEST_CASE_BODY(build_c_o)
193{
194    ATF_TEST_CASE_USE(h_build_c_o_ok);
195    run_h_tc< ATF_TEST_CASE_NAME(h_build_c_o_ok) >();
196    ATF_REQUIRE(atf::utils::grep_file("-o test.o", "stdout"));
197    ATF_REQUIRE(atf::utils::grep_file("-c test.c", "stdout"));
198
199    ATF_TEST_CASE_USE(h_build_c_o_fail);
200    run_h_tc< ATF_TEST_CASE_NAME(h_build_c_o_fail) >();
201    ATF_REQUIRE(atf::utils::grep_file("-o test.o", "stdout"));
202    ATF_REQUIRE(atf::utils::grep_file("-c test.c", "stdout"));
203    ATF_REQUIRE(atf::utils::grep_file("test.c", "stderr"));
204    ATF_REQUIRE(atf::utils::grep_file("UNDEFINED_SYMBOL", "stderr"));
205}
206
207ATF_TEST_CASE(build_cpp);
208ATF_TEST_CASE_HEAD(build_cpp)
209{
210    set_md_var("descr", "Tests the build_cpp function");
211}
212ATF_TEST_CASE_BODY(build_cpp)
213{
214    ATF_TEST_CASE_USE(h_build_cpp_ok);
215    run_h_tc< ATF_TEST_CASE_NAME(h_build_cpp_ok) >();
216    ATF_REQUIRE(atf::utils::grep_file("-o.*test.p", "stdout"));
217    ATF_REQUIRE(atf::utils::grep_file("test.c", "stdout"));
218    ATF_REQUIRE(atf::utils::grep_file("foo bar", "test.p"));
219
220    ATF_TEST_CASE_USE(h_build_cpp_fail);
221    run_h_tc< ATF_TEST_CASE_NAME(h_build_cpp_fail) >();
222    ATF_REQUIRE(atf::utils::grep_file("-o test.p", "stdout"));
223    ATF_REQUIRE(atf::utils::grep_file("test.c", "stdout"));
224    ATF_REQUIRE(atf::utils::grep_file("test.c", "stderr"));
225    ATF_REQUIRE(atf::utils::grep_file("non-existent.h", "stderr"));
226}
227
228ATF_TEST_CASE(build_cxx_o);
229ATF_TEST_CASE_HEAD(build_cxx_o)
230{
231    set_md_var("descr", "Tests the build_cxx_o function");
232}
233ATF_TEST_CASE_BODY(build_cxx_o)
234{
235    ATF_TEST_CASE_USE(h_build_cxx_o_ok);
236    run_h_tc< ATF_TEST_CASE_NAME(h_build_cxx_o_ok) >();
237    ATF_REQUIRE(atf::utils::grep_file("-o test.o", "stdout"));
238    ATF_REQUIRE(atf::utils::grep_file("-c test.cpp", "stdout"));
239
240    ATF_TEST_CASE_USE(h_build_cxx_o_fail);
241    run_h_tc< ATF_TEST_CASE_NAME(h_build_cxx_o_fail) >();
242    ATF_REQUIRE(atf::utils::grep_file("-o test.o", "stdout"));
243    ATF_REQUIRE(atf::utils::grep_file("-c test.cpp", "stdout"));
244    ATF_REQUIRE(atf::utils::grep_file("test.cpp", "stderr"));
245    ATF_REQUIRE(atf::utils::grep_file("UNDEFINED_SYMBOL", "stderr"));
246}
247
248ATF_TEST_CASE(exec_cleanup);
249ATF_TEST_CASE_HEAD(exec_cleanup)
250{
251    set_md_var("descr", "Tests that exec properly cleans up the temporary "
252               "files it creates");
253}
254ATF_TEST_CASE_BODY(exec_cleanup)
255{
256    std::auto_ptr< atf::fs::path > out;
257    std::auto_ptr< atf::fs::path > err;
258
259    {
260        std::auto_ptr< atf::check::check_result > r =
261            do_exec(this, "exit-success");
262        out.reset(new atf::fs::path(r->stdout_path()));
263        err.reset(new atf::fs::path(r->stderr_path()));
264        ATF_REQUIRE(atf::fs::exists(*out.get()));
265        ATF_REQUIRE(atf::fs::exists(*err.get()));
266    }
267    ATF_REQUIRE(!atf::fs::exists(*out.get()));
268    ATF_REQUIRE(!atf::fs::exists(*err.get()));
269}
270
271ATF_TEST_CASE(exec_exitstatus);
272ATF_TEST_CASE_HEAD(exec_exitstatus)
273{
274    set_md_var("descr", "Tests that exec properly captures the exit "
275               "status of the executed command");
276}
277ATF_TEST_CASE_BODY(exec_exitstatus)
278{
279    {
280        std::auto_ptr< atf::check::check_result > r =
281            do_exec(this, "exit-success");
282        ATF_REQUIRE(r->exited());
283        ATF_REQUIRE(!r->signaled());
284        ATF_REQUIRE_EQ(r->exitcode(), EXIT_SUCCESS);
285    }
286
287    {
288        std::auto_ptr< atf::check::check_result > r =
289            do_exec(this, "exit-failure");
290        ATF_REQUIRE(r->exited());
291        ATF_REQUIRE(!r->signaled());
292        ATF_REQUIRE_EQ(r->exitcode(), EXIT_FAILURE);
293    }
294
295    {
296        std::auto_ptr< atf::check::check_result > r =
297            do_exec(this, "exit-signal");
298        ATF_REQUIRE(!r->exited());
299        ATF_REQUIRE(r->signaled());
300        ATF_REQUIRE_EQ(r->termsig(), SIGKILL);
301    }
302}
303
304static
305void
306check_lines(const std::string& path, const char* outname,
307            const char* resname)
308{
309    std::ifstream f(path.c_str());
310    ATF_REQUIRE(f);
311
312    std::string line;
313    std::getline(f, line);
314    ATF_REQUIRE_EQ(line, std::string("Line 1 to ") + outname + " for " +
315                    resname);
316    std::getline(f, line);
317    ATF_REQUIRE_EQ(line, std::string("Line 2 to ") + outname + " for " +
318                    resname);
319}
320
321ATF_TEST_CASE(exec_stdout_stderr);
322ATF_TEST_CASE_HEAD(exec_stdout_stderr)
323{
324    set_md_var("descr", "Tests that exec properly captures the stdout "
325               "and stderr streams of the child process");
326}
327ATF_TEST_CASE_BODY(exec_stdout_stderr)
328{
329    std::auto_ptr< atf::check::check_result > r1 =
330        do_exec(this, "stdout-stderr", "result1");
331    ATF_REQUIRE(r1->exited());
332    ATF_REQUIRE_EQ(r1->exitcode(), EXIT_SUCCESS);
333
334    std::auto_ptr< atf::check::check_result > r2 =
335        do_exec(this, "stdout-stderr", "result2");
336    ATF_REQUIRE(r2->exited());
337    ATF_REQUIRE_EQ(r2->exitcode(), EXIT_SUCCESS);
338
339    const std::string out1 = r1->stdout_path();
340    const std::string out2 = r2->stdout_path();
341    const std::string err1 = r1->stderr_path();
342    const std::string err2 = r2->stderr_path();
343
344    ATF_REQUIRE(out1.find("check.XXXXXX") == std::string::npos);
345    ATF_REQUIRE(out2.find("check.XXXXXX") == std::string::npos);
346    ATF_REQUIRE(err1.find("check.XXXXXX") == std::string::npos);
347    ATF_REQUIRE(err2.find("check.XXXXXX") == std::string::npos);
348
349    ATF_REQUIRE(out1.find("/check") != std::string::npos);
350    ATF_REQUIRE(out2.find("/check") != std::string::npos);
351    ATF_REQUIRE(err1.find("/check") != std::string::npos);
352    ATF_REQUIRE(err2.find("/check") != std::string::npos);
353
354    ATF_REQUIRE(out1.find("/stdout") != std::string::npos);
355    ATF_REQUIRE(out2.find("/stdout") != std::string::npos);
356    ATF_REQUIRE(err1.find("/stderr") != std::string::npos);
357    ATF_REQUIRE(err2.find("/stderr") != std::string::npos);
358
359    ATF_REQUIRE(out1 != out2);
360    ATF_REQUIRE(err1 != err2);
361
362    check_lines(out1, "stdout", "result1");
363    check_lines(out2, "stdout", "result2");
364    check_lines(err1, "stderr", "result1");
365    check_lines(err2, "stderr", "result2");
366}
367
368ATF_TEST_CASE(exec_unknown);
369ATF_TEST_CASE_HEAD(exec_unknown)
370{
371    set_md_var("descr", "Tests that running a non-existing binary "
372               "is handled correctly");
373}
374ATF_TEST_CASE_BODY(exec_unknown)
375{
376    std::vector< std::string > argv;
377    argv.push_back(atf::config::get("atf_workdir") + "/non-existent");
378
379    atf::process::argv_array argva(argv);
380    std::auto_ptr< atf::check::check_result > r = atf::check::exec(argva);
381    ATF_REQUIRE(r->exited());
382    ATF_REQUIRE_EQ(r->exitcode(), 127);
383}
384
385// ------------------------------------------------------------------------
386// Tests cases for the header file.
387// ------------------------------------------------------------------------
388
389HEADER_TC(include, "atf-c++/check.hpp");
390
391// ------------------------------------------------------------------------
392// Main.
393// ------------------------------------------------------------------------
394
395ATF_INIT_TEST_CASES(tcs)
396{
397    // Add the test cases for the free functions.
398    ATF_ADD_TEST_CASE(tcs, build_c_o);
399    ATF_ADD_TEST_CASE(tcs, build_cpp);
400    ATF_ADD_TEST_CASE(tcs, build_cxx_o);
401    ATF_ADD_TEST_CASE(tcs, exec_cleanup);
402    ATF_ADD_TEST_CASE(tcs, exec_exitstatus);
403    ATF_ADD_TEST_CASE(tcs, exec_stdout_stderr);
404    ATF_ADD_TEST_CASE(tcs, exec_unknown);
405
406    // Add the test cases for the header file.
407    ATF_ADD_TEST_CASE(tcs, include);
408}
409