atf-report_test.sh revision 4a53e3c2b83c476a93148eaee0272649beb221ca
#
# Automated Testing Framework (atf)
#
# Copyright (c) 2007 The NetBSD Foundation, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
#
# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
# CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
{
#! $(atf-config -t atf_shell)
echo foo
EOF
Content-Type: application/X-atf-atffile; version="1"
prop: test-suite = atf
tp: dir1
tp: tp3
tp: tp4
tp: tp5
EOF
Content-Type: application/X-atf-atffile; version="1"
prop: test-suite = atf
tp: tp1
tp: tp2
EOF
}
{
cat >etc/atf-run.hooks <<EOF
#! $(atf-config -t atf_shell)
info_start_hook()
{
atf_tps_writer_info "startinfo" "A value"
}
info_end_hook()
{
atf_tps_writer_info "endinfo" "Another value"
}
EOF
echo "Using atf-run to run helpers"
}
{
"format"
}
{
# Check that the default output uses the ticker format.
}
# XXX The test for all expect_ values should be intermixed with the other
# tests. However, to do that, we need to migrate to using C helpers for
# simplicity in raising signals...
{
Content-Type: application/X-atf-atffile; version="1"
prop: test-suite = atf
tp: expect_helpers
EOF
# NO_CHECK_STYLE_BEGIN
tc, #.#, expect_helpers, death_and_exit, expected_death, Exit case
tc, #.#, expect_helpers, death_and_signal, expected_death, Signal case
tc, #.#, expect_helpers, death_but_pass, failed, Test case was expected to terminate abruptly but it continued execution
tc, #.#, expect_helpers, exit_any_and_exit, expected_exit, Call will exit
tc, #.#, expect_helpers, exit_but_pass, failed, Test case was expected to exit cleanly but it continued execution
tc, #.#, expect_helpers, exit_code_and_exit, expected_exit, Call will exit
tc, #.#, expect_helpers, fail_and_fail_check, expected_failure, And fail again: 2 checks failed as expected; see output for more details
tc, #.#, expect_helpers, fail_and_fail_requirement, expected_failure, Fail reason: The failure
tc, #.#, expect_helpers, fail_but_pass, failed, Test case was expecting a failure but none were raised
tc, #.#, expect_helpers, pass_and_pass, passed
tc, #.#, expect_helpers, pass_but_fail_check, failed, 1 checks failed; see output for more details
tc, #.#, expect_helpers, pass_but_fail_requirement, failed, Some reason
tc, #.#, expect_helpers, signal_any_and_signal, expected_signal, Call will signal
tc, #.#, expect_helpers, signal_but_pass, failed, Test case was expected to receive a termination signal but it continued execution
tc, #.#, expect_helpers, signal_no_and_signal, expected_signal, Call will signal
tc, #.#, expect_helpers, timeout_and_hang, expected_timeout, Will overrun
tc, #.#, expect_helpers, timeout_but_pass, failed, Test case was expected to hang but it continued execution
tp, #.#, expect_helpers, failed
EOF
# NO_CHECK_STYLE_END
"sed -E -e 's/[0-9]+.[0-9]{6}, /#.#, /'"
# NO_CHECK_STYLE_BEGIN
expect_helpers (1/1): 17 test cases
death_and_exit: [#.#s] Expected failure: Exit case
death_and_signal: [#.#s] Expected failure: Signal case
death_but_pass: [#.#s] Failed: Test case was expected to terminate abruptly but it continued execution
exit_any_and_exit: [#.#s] Expected failure: Call will exit
exit_but_pass: [#.#s] Failed: Test case was expected to exit cleanly but it continued execution
exit_code_and_exit: [#.#s] Expected failure: Call will exit
fail_and_fail_check: [#.#s] Expected failure: And fail again: 2 checks failed as expected; see output for more details
fail_and_fail_requirement: [#.#s] Expected failure: Fail reason: The failure
fail_but_pass: [#.#s] Failed: Test case was expecting a failure but none were raised
pass_and_pass: [#.#s] Passed.
pass_but_fail_check: [#.#s] Failed: 1 checks failed; see output for more details
pass_but_fail_requirement: [#.#s] Failed: Some reason
signal_any_and_signal: [#.#s] Expected failure: Call will signal
signal_but_pass: [#.#s] Failed: Test case was expected to receive a termination signal but it continued execution
signal_no_and_signal: [#.#s] Expected failure: Call will signal
timeout_and_hang: [#.#s] Expected failure: Will overrun
timeout_but_pass: [#.#s] Failed: Test case was expected to hang but it continued execution
[#.#s]
Test cases for known bugs:
expect_helpers:death_and_exit: Exit case
expect_helpers:death_and_signal: Signal case
expect_helpers:exit_any_and_exit: Call will exit
expect_helpers:exit_code_and_exit: Call will exit
expect_helpers:fail_and_fail_check: And fail again: 2 checks failed as expected; see output for more details
expect_helpers:fail_and_fail_requirement: Fail reason: The failure
expect_helpers:signal_any_and_signal: Call will signal
expect_helpers:signal_no_and_signal: Call will signal
expect_helpers:timeout_and_hang: Will overrun
Failed test cases:
expect_helpers:death_but_pass, expect_helpers:exit_but_pass, expect_helpers:fail_but_pass, expect_helpers:pass_but_fail_check, expect_helpers:pass_but_fail_requirement, expect_helpers:signal_but_pass, expect_helpers:timeout_but_pass
Summary for 1 test programs:
1 passed test cases.
7 failed test cases.
9 expected failed test cases.
0 skipped test cases.
EOF
# NO_CHECK_STYLE_END
"sed -E -e 's/[0-9]+.[0-9]{6}/#.#/'"
# Just ensure that this does not crash for now...
}
{
atf_set "descr" "Checks that the -o flag works"
}
{
# Get the default output.
# Check that changing the stdout output works.
# Check that sending the output to a file does not write to stdout.
# Check that defining two outputs using the same format works.
# Check that defining two outputs using different formats works.
# Check that defining two outputs over the same file does not work.
# Check that defining two outputs over stdout (but using different
# paths) does not work.
}
{
atf_set "descr" "Checks the CSV output format"
}
{
# NO_CHECK_STYLE_BEGIN
tc, #.#, dir1/tp1, main, passed
tp, #.#, dir1/tp1, passed
tc, #.#, dir1/tp2, main, failed, This always fails
tp, #.#, dir1/tp2, failed
tc, #.#, tp3, main, passed
tp, #.#, tp3, passed
tc, #.#, tp4, main, failed, This always fails
tp, #.#, tp4, failed
tp, #.#, tp5, bogus, Invalid format for test case list: 1: Unexpected token \`<<NEWLINE>>'; expected \`:'
EOF
# NO_CHECK_STYLE_END
"atf-report -o csv:- <tps.out | sed -E -e 's/[0-9]+.[0-9]{6}, /#.#, /'"
}
{
atf_set "descr" "Checks the ticker output format"
}
{
# NO_CHECK_STYLE_BEGIN
dir1/tp1 (1/5): 1 test cases
main: [#.#s] Passed.
[#.#s]
dir1/tp2 (2/5): 1 test cases
main: [#.#s] Failed: This always fails
[#.#s]
tp3 (3/5): 1 test cases
main: [#.#s] Passed.
[#.#s]
tp4 (4/5): 1 test cases
main: [#.#s] Failed: This always fails
[#.#s]
tp5 (5/5): 0 test cases
tp5: BOGUS TEST PROGRAM: Cannot trust its results because of \`Invalid format for test case list: 1: Unexpected token \`<<NEWLINE>>'; expected \`:''
[#.#s]
Failed (bogus) test programs:
tp5
Failed test cases:
dir1/tp2:main, tp4:main
Summary for 5 test programs:
2 passed test cases.
2 failed test cases.
0 expected failed test cases.
0 skipped test cases.
EOF
"atf-report -o ticker:- <tps.out | sed -E -e 's/[0-9]+.[0-9]{6}/#.#/'"
}
# NO_CHECK_STYLE_END
{
atf_set "descr" "Checks the XML output format"
}
{
# NO_CHECK_STYLE_BEGIN
<?xml version="1.0" encoding="ISO-8859-1"?>
<!DOCTYPE tests-results PUBLIC "-//NetBSD//DTD ATF Tests Results 0.1//EN" "http://www.NetBSD.org/XML/atf/tests-results.dtd">
<tests-results>
<info class="startinfo">A value</info>
<tp id="dir1/tp1">
<tc id="main">
<passed />
<tc-time>#.#</tc-time></tc>
<tp-time>#.#</tp-time></tp>
<tp id="dir1/tp2">
<tc id="main">
<failed>This always fails</failed>
<tc-time>#.#</tc-time></tc>
<tp-time>#.#</tp-time></tp>
<tp id="tp3">
<tc id="main">
<passed />
<tc-time>#.#</tc-time></tc>
<tp-time>#.#</tp-time></tp>
<tp id="tp4">
<tc id="main">
<failed>This always fails</failed>
<tc-time>#.#</tc-time></tc>
<tp-time>#.#</tp-time></tp>
<tp id="tp5">
<failed>Invalid format for test case list: 1: Unexpected token \`<<NEWLINE>>'; expected \`:'</failed>
<tp-time>#.#</tp-time></tp>
<info class="endinfo">Another value</info>
</tests-results>
EOF
# NO_CHECK_STYLE_END
"atf-report -o xml:- < tps.out | sed -E -e 's/>[0-9]+.[0-9]{6}</>#.#</'"
}
{
"leading and trailing whitespace in stdout and stderr" \
"lines"
}
{
Content-Type: application/X-atf-atffile; version="1"
prop: test-suite = atf
tp: misc_helpers
EOF
# NO_CHECK_STYLE_BEGIN
<?xml version="1.0" encoding="ISO-8859-1"?>
<!DOCTYPE tests-results PUBLIC "-//NetBSD//DTD ATF Tests Results 0.1//EN" "http://www.NetBSD.org/XML/atf/tests-results.dtd">
<tests-results>
<info class="startinfo">A value</info>
<tp id="misc_helpers">
<tc id="diff">
<so>--- a 2007-11-04 14:00:41.000000000 +0100</so>
<so>+++ b 2007-11-04 14:00:48.000000000 +0100</so>
<so>@@ -1,7 +1,7 @@</so>
<so> This test is meant to simulate a diff.</so>
<so> Blank space at beginning of context lines must be preserved.</so>
<so> </so>
<so>-First original line.</so>
<so>-Second original line.</so>
<so>+First modified line.</so>
<so>+Second modified line.</so>
<so> </so>
<so> EOF</so>
<passed />
<tc-time>#.#</tc-time></tc>
<tp-time>#.#</tp-time></tp>
<info class="endinfo">Another value</info>
</tests-results>
EOF
# NO_CHECK_STYLE_END
"atf-report -o xml:- <tps.out | sed -E -e 's/>[0-9]+.[0-9]{6}</>#.#</'"
}
{
atf-report: ERROR: No arguments allowed
EOF
}
{
}
# vim: syntax=sh:expandtab:shiftwidth=4:softtabstop=4