diff options
Diffstat (limited to '')
| -rw-r--r-- | unit/atf-src/tools/atf-run.cpp | 567 |
1 files changed, 567 insertions, 0 deletions
diff --git a/unit/atf-src/tools/atf-run.cpp b/unit/atf-src/tools/atf-run.cpp new file mode 100644 index 0000000..a05b681 --- /dev/null +++ b/unit/atf-src/tools/atf-run.cpp @@ -0,0 +1,567 @@ +// +// Automated Testing Framework (atf) +// +// Copyright (c) 2007 The NetBSD Foundation, Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND +// CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, +// INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +// IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +// GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +// IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +// IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#if defined(HAVE_CONFIG_H) +#include "config.h" +#endif + +extern "C" { +#include <sys/types.h> +#include <sys/param.h> +#include <sys/stat.h> +#include <sys/wait.h> +#include <unistd.h> +} + +#include <algorithm> +#include <cassert> +#include <cerrno> +#include <cstdlib> +#include <cstring> +#include <fstream> +#include <iostream> +#include <map> +#include <string> + +#include "application.hpp" +#include "atffile.hpp" +#include "config.hpp" +#include "config_file.hpp" +#include "env.hpp" +#include "exceptions.hpp" +#include "fs.hpp" +#include "parser.hpp" +#include "process.hpp" +#include "requirements.hpp" +#include "test-program.hpp" +#include "text.hpp" + +namespace { + +typedef std::map< std::string, std::string > vars_map; + +} // anonymous namespace + +#if defined(MAXCOMLEN) +static const std::string::size_type max_core_name_length = MAXCOMLEN; +#else +static const std::string::size_type max_core_name_length = std::string::npos; +#endif + +class atf_run : public tools::application::app { + static const char* m_description; + + vars_map m_cmdline_vars; + + static vars_map::value_type parse_var(const std::string&); + + void process_option(int, const char*); + std::string specific_args(void) const; + options_set specific_options(void) const; + + void parse_vflag(const std::string&); + + std::vector< std::string > conf_args(void) const; + + size_t count_tps(std::vector< std::string >) const; + + int run_test(const tools::fs::path&, tools::test_program::atf_tps_writer&, + const vars_map&); + int run_test_directory(const tools::fs::path&, + tools::test_program::atf_tps_writer&); + int run_test_program(const tools::fs::path&, + tools::test_program::atf_tps_writer&, + const vars_map&); + + tools::test_program::test_case_result get_test_case_result( + const std::string&, const tools::process::status&, + const tools::fs::path&) const; + +public: + atf_run(void); + + int main(void); +}; + +static void +sanitize_gdb_env(void) +{ + try { + tools::env::unset("TERM"); + } catch (...) { + // Just swallow exceptions here; they cannot propagate into C, which + // is where this function is called from, and even if these exceptions + // appear they are benign. + } +} + +static void +dump_stacktrace(const tools::fs::path& tp, const tools::process::status& s, + const tools::fs::path& workdir, + tools::test_program::atf_tps_writer& w) +{ + assert(s.signaled() && s.coredump()); + + w.stderr_tc("Test program crashed; attempting to get stack trace"); + + const tools::fs::path corename = workdir / + (tp.leaf_name().substr(0, max_core_name_length) + ".core"); + if (!tools::fs::exists(corename)) { + w.stderr_tc("Expected file " + corename.str() + " not found"); + return; + } + + const tools::fs::path gdb(GDB); + const tools::fs::path gdbout = workdir / "gdb.out"; + const tools::process::argv_array args(gdb.leaf_name().c_str(), "-batch", + "-q", "-ex", "bt", tp.c_str(), + corename.c_str(), NULL); + tools::process::status status = tools::process::exec( + gdb, args, + tools::process::stream_redirect_path(gdbout), + tools::process::stream_redirect_path(tools::fs::path("/dev/null")), + sanitize_gdb_env); + if (!status.exited() || status.exitstatus() != EXIT_SUCCESS) { + w.stderr_tc("Execution of " GDB " failed"); + return; + } + + std::ifstream input(gdbout.c_str()); + if (input) { + std::string line; + while (std::getline(input, line).good()) + w.stderr_tc(line); + input.close(); + } + + w.stderr_tc("Stack trace complete"); +} + +const char* atf_run::m_description = + "atf-run is a tool that runs tests programs and collects their " + "results."; + +atf_run::atf_run(void) : + app(m_description, "atf-run(1)", "atf(7)") +{ +} + +void +atf_run::process_option(int ch, const char* arg) +{ + switch (ch) { + case 'v': + parse_vflag(arg); + break; + + default: + std::abort(); + } +} + +std::string +atf_run::specific_args(void) + const +{ + return "[test-program1 .. test-programN]"; +} + +atf_run::options_set +atf_run::specific_options(void) + const +{ + using tools::application::option; + options_set opts; + opts.insert(option('v', "var=value", "Sets the configuration variable " + "`var' to `value'; overrides " + "values in configuration files")); + return opts; +} + +void +atf_run::parse_vflag(const std::string& str) +{ + if (str.empty()) + throw std::runtime_error("-v requires a non-empty argument"); + + std::vector< std::string > ws = tools::text::split(str, "="); + if (ws.size() == 1 && str[str.length() - 1] == '=') { + m_cmdline_vars[ws[0]] = ""; + } else { + if (ws.size() != 2) + throw std::runtime_error("-v requires an argument of the form " + "var=value"); + + m_cmdline_vars[ws[0]] = ws[1]; + } +} + +int +atf_run::run_test(const tools::fs::path& tp, + tools::test_program::atf_tps_writer& w, + const vars_map& config) +{ + tools::fs::file_info fi(tp); + + int errcode; + if (fi.get_type() == tools::fs::file_info::dir_type) + errcode = run_test_directory(tp, w); + else { + const vars_map effective_config = + tools::config_file::merge_configs(config, m_cmdline_vars); + + errcode = run_test_program(tp, w, effective_config); + } + return errcode; +} + +int +atf_run::run_test_directory(const tools::fs::path& tp, + tools::test_program::atf_tps_writer& w) +{ + tools::atffile af = tools::read_atffile(tp / "Atffile"); + + vars_map test_suite_vars; + { + vars_map::const_iterator iter = af.props().find("test-suite"); + assert(iter != af.props().end()); + test_suite_vars = tools::config_file::read_config_files((*iter).second); + } + + bool ok = true; + for (std::vector< std::string >::const_iterator iter = af.tps().begin(); + iter != af.tps().end(); iter++) { + const bool result = run_test(tp / *iter, w, + tools::config_file::merge_configs(af.conf(), test_suite_vars)); + ok &= (result == EXIT_SUCCESS); + } + + return ok ? EXIT_SUCCESS : EXIT_FAILURE; +} + +tools::test_program::test_case_result +atf_run::get_test_case_result(const std::string& broken_reason, + const tools::process::status& s, + const tools::fs::path& resfile) + const +{ + using tools::text::to_string; + using tools::test_program::read_test_case_result; + using tools::test_program::test_case_result; + + if (!broken_reason.empty()) { + test_case_result tcr; + + try { + tcr = read_test_case_result(resfile); + + if (tcr.state() == "expected_timeout") { + return tcr; + } else { + return test_case_result("failed", -1, broken_reason); + } + } catch (const std::runtime_error&) { + return test_case_result("failed", -1, broken_reason); + } + } + + if (s.exited()) { + test_case_result tcr; + + try { + tcr = read_test_case_result(resfile); + } catch (const std::runtime_error& e) { + return test_case_result("failed", -1, "Test case exited " + "normally but failed to create the results file: " + + std::string(e.what())); + } + + if (tcr.state() == "expected_death") { + return tcr; + } else if (tcr.state() == "expected_exit") { + if (tcr.value() == -1 || s.exitstatus() == tcr.value()) + return tcr; + else + return test_case_result("failed", -1, "Test case was " + "expected to exit with a " + to_string(tcr.value()) + + " error code but returned " + to_string(s.exitstatus())); + } else if (tcr.state() == "expected_failure") { + if (s.exitstatus() == EXIT_SUCCESS) + return tcr; + else + return test_case_result("failed", -1, "Test case returned an " + "error in expected_failure mode but it should not have"); + } else if (tcr.state() == "expected_signal") { + return test_case_result("failed", -1, "Test case exited cleanly " + "but was expected to receive a signal"); + } else if (tcr.state() == "failed") { + if (s.exitstatus() == EXIT_SUCCESS) + return test_case_result("failed", -1, "Test case " + "exited successfully but reported failure"); + else + return tcr; + } else if (tcr.state() == "passed") { + if (s.exitstatus() == EXIT_SUCCESS) + return tcr; + else + return test_case_result("failed", -1, "Test case exited as " + "passed but reported an error"); + } else if (tcr.state() == "skipped") { + if (s.exitstatus() == EXIT_SUCCESS) + return tcr; + else + return test_case_result("failed", -1, "Test case exited as " + "skipped but reported an error"); + } + } else if (s.signaled()) { + test_case_result tcr; + + try { + tcr = read_test_case_result(resfile); + } catch (const std::runtime_error&) { + return test_case_result("failed", -1, "Test program received " + "signal " + tools::text::to_string(s.termsig()) + + (s.coredump() ? " (core dumped)" : "")); + } + + if (tcr.state() == "expected_death") { + return tcr; + } else if (tcr.state() == "expected_signal") { + if (tcr.value() == -1 || s.termsig() == tcr.value()) + return tcr; + else + return test_case_result("failed", -1, "Test case was " + "expected to exit due to a " + to_string(tcr.value()) + + " signal but got " + to_string(s.termsig())); + } else { + return test_case_result("failed", -1, "Test program received " + "signal " + tools::text::to_string(s.termsig()) + + (s.coredump() ? " (core dumped)" : "") + " and created a " + "bogus results file"); + } + } + std::abort(); + return test_case_result(); +} + +int +atf_run::run_test_program(const tools::fs::path& tp, + tools::test_program::atf_tps_writer& w, + const vars_map& config) +{ + int errcode = EXIT_SUCCESS; + + tools::test_program::metadata md; + try { + md = tools::test_program::get_metadata(tp, config); + } catch (const tools::parser::format_error& e) { + w.start_tp(tp.str(), 0); + w.end_tp("Invalid format for test case list: " + std::string(e.what())); + return EXIT_FAILURE; + } catch (const tools::parser::parse_errors& e) { + const std::string reason = tools::text::join(e, "; "); + w.start_tp(tp.str(), 0); + w.end_tp("Invalid format for test case list: " + reason); + return EXIT_FAILURE; + } + + tools::fs::temp_dir resdir( + tools::fs::path(tools::config::get("atf_workdir")) / "atf-run.XXXXXX"); + + w.start_tp(tp.str(), md.test_cases.size()); + if (md.test_cases.empty()) { + w.end_tp("Bogus test program: reported 0 test cases"); + errcode = EXIT_FAILURE; + } else { + for (std::map< std::string, vars_map >::const_iterator iter + = md.test_cases.begin(); iter != md.test_cases.end(); iter++) { + const std::string& tcname = (*iter).first; + const vars_map& tcmd = (*iter).second; + + w.start_tc(tcname); + + try { + const std::string& reqfail = tools::check_requirements( + tcmd, config); + if (!reqfail.empty()) { + w.end_tc("skipped", reqfail); + continue; + } + } catch (const std::runtime_error& e) { + w.end_tc("failed", e.what()); + errcode = EXIT_FAILURE; + continue; + } + + const std::pair< int, int > user = tools::get_required_user( + tcmd, config); + + tools::fs::path resfile = resdir.get_path() / "tcr"; + assert(!tools::fs::exists(resfile)); + try { + const bool has_cleanup = tools::text::to_bool( + (*tcmd.find("has.cleanup")).second); + + tools::fs::temp_dir workdir(tools::fs::path(tools::config::get( + "atf_workdir")) / "atf-run.XXXXXX"); + if (user.first != -1 && user.second != -1) { + if (::chown(workdir.get_path().c_str(), user.first, + user.second) == -1) { + throw tools::system_error("chown(" + + workdir.get_path().str() + ")", "chown(2) failed", + errno); + } + resfile = workdir.get_path() / "tcr"; + } + + std::pair< std::string, const tools::process::status > s = + tools::test_program::run_test_case( + tp, tcname, "body", tcmd, config, + resfile, workdir.get_path(), w); + if (s.second.signaled() && s.second.coredump()) + dump_stacktrace(tp, s.second, workdir.get_path(), w); + if (has_cleanup) + (void)tools::test_program::run_test_case( + tp, tcname, "cleanup", tcmd, + config, resfile, workdir.get_path(), w); + + // TODO: Force deletion of workdir. + + tools::test_program::test_case_result tcr = + get_test_case_result(s.first, s.second, resfile); + + w.end_tc(tcr.state(), tcr.reason()); + if (tcr.state() == "failed") + errcode = EXIT_FAILURE; + } catch (...) { + if (tools::fs::exists(resfile)) + tools::fs::remove(resfile); + throw; + } + if (tools::fs::exists(resfile)) + tools::fs::remove(resfile); + + } + w.end_tp(""); + } + + return errcode; +} + +size_t +atf_run::count_tps(std::vector< std::string > tps) + const +{ + size_t ntps = 0; + + for (std::vector< std::string >::const_iterator iter = tps.begin(); + iter != tps.end(); iter++) { + tools::fs::path tp(*iter); + tools::fs::file_info fi(tp); + + if (fi.get_type() == tools::fs::file_info::dir_type) { + tools::atffile af = tools::read_atffile(tp / "Atffile"); + std::vector< std::string > aux = af.tps(); + for (std::vector< std::string >::iterator i2 = aux.begin(); + i2 != aux.end(); i2++) + *i2 = (tp / *i2).str(); + ntps += count_tps(aux); + } else + ntps++; + } + + return ntps; +} + +static +void +call_hook(const std::string& tool, const std::string& hook) +{ + const tools::fs::path sh(tools::config::get("atf_shell")); + const tools::fs::path hooks = + tools::fs::path(tools::config::get("atf_pkgdatadir")) / (tool + ".hooks"); + + const tools::process::status s = + tools::process::exec(sh, + tools::process::argv_array(sh.c_str(), hooks.c_str(), + hook.c_str(), NULL), + tools::process::stream_inherit(), + tools::process::stream_inherit()); + + + if (!s.exited() || s.exitstatus() != EXIT_SUCCESS) + throw std::runtime_error("Failed to run the '" + hook + "' hook " + "for '" + tool + "'"); +} + +int +atf_run::main(void) +{ + tools::atffile af = tools::read_atffile(tools::fs::path("Atffile")); + + std::vector< std::string > tps; + tps = af.tps(); + if (m_argc >= 1) { + // TODO: Ensure that the given test names are listed in the + // Atffile. Take into account that the file can be using globs. + tps.clear(); + for (int i = 0; i < m_argc; i++) + tps.push_back(m_argv[i]); + } + + // Read configuration data for this test suite. + vars_map test_suite_vars; + { + vars_map::const_iterator iter = af.props().find("test-suite"); + assert(iter != af.props().end()); + test_suite_vars = tools::config_file::read_config_files((*iter).second); + } + + tools::test_program::atf_tps_writer w(std::cout); + call_hook("atf-run", "info_start_hook"); + w.ntps(count_tps(tps)); + + bool ok = true; + for (std::vector< std::string >::const_iterator iter = tps.begin(); + iter != tps.end(); iter++) { + const bool result = run_test(tools::fs::path(*iter), w, + tools::config_file::merge_configs(af.conf(), test_suite_vars)); + ok &= (result == EXIT_SUCCESS); + } + + call_hook("atf-run", "info_end_hook"); + + return ok ? EXIT_SUCCESS : EXIT_FAILURE; +} + +int +main(int argc, char* const* argv) +{ + return atf_run().run(argc, argv); +} |