summaryrefslogtreecommitdiffstats
path: root/unit/atf-src/tools/atf-report_test.sh
blob: 9f0f47fc61cf998e694563c4ee2b9ace231eff18 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
#
# Automated Testing Framework (atf)
#
# Copyright (c) 2007 The NetBSD Foundation, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND
# CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
# IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

create_helpers()
{
    mkdir dir1
    cp $(atf_get_srcdir)/pass_helper dir1/tp1
    cp $(atf_get_srcdir)/fail_helper dir1/tp2
    cp $(atf_get_srcdir)/pass_helper tp3
    cp $(atf_get_srcdir)/fail_helper tp4

    cat >tp5 <<EOF
#! $(atf-config -t atf_shell)
echo foo
EOF
    chmod +x tp5

    cat >Atffile <<EOF
Content-Type: application/X-atf-atffile; version="1"

prop: test-suite = atf

tp: dir1
tp: tp3
tp: tp4
tp: tp5
EOF

    cat >dir1/Atffile <<EOF
Content-Type: application/X-atf-atffile; version="1"

prop: test-suite = atf

tp: tp1
tp: tp2
EOF
}

run_helpers()
{
    mkdir etc
    cat >etc/atf-run.hooks <<EOF
#! $(atf-config -t atf_shell)

info_start_hook()
{
    atf_tps_writer_info "startinfo" "A value"
}

info_end_hook()
{
    atf_tps_writer_info "endinfo" "Another value"
}
EOF
    echo "Using atf-run to run helpers"
    ATF_CONFDIR=$(pwd)/etc atf-run >tps.out 2>/dev/null
    rm -rf etc
}

atf_test_case default
default_head()
{
    atf_set "descr" "Checks that the default output uses the ticker" \
                    "format"
}
default_body()
{
    create_helpers
    run_helpers

    # Check that the default output uses the ticker format.
    atf_check -s eq:0 -o match:'test cases' -o match:'Failed test cases' \
        -o match:'Summary for' -e empty -x 'atf-report <tps.out'
}

# XXX The test for all expect_ values should be intermixed with the other
# tests.  However, to do that, we need to migrate to using C helpers for
# simplicity in raising signals...
atf_test_case expect
expect_body()
{
    ln -s "$(atf_get_srcdir)/expect_helpers" .
    cat >Atffile <<EOF
Content-Type: application/X-atf-atffile; version="1"

prop: test-suite = atf

tp: expect_helpers
EOF
    run_helpers

# NO_CHECK_STYLE_BEGIN
    cat >expout <<EOF
tc, #.#, expect_helpers, death_and_exit, expected_death, Exit case
tc, #.#, expect_helpers, death_and_signal, expected_death, Signal case
tc, #.#, expect_helpers, death_but_pass, failed, Test case was expected to terminate abruptly but it continued execution
tc, #.#, expect_helpers, exit_any_and_exit, expected_exit, Call will exit
tc, #.#, expect_helpers, exit_but_pass, failed, Test case was expected to exit cleanly but it continued execution
tc, #.#, expect_helpers, exit_code_and_exit, expected_exit, Call will exit
tc, #.#, expect_helpers, fail_and_fail_check, expected_failure, And fail again: 2 checks failed as expected; see output for more details
tc, #.#, expect_helpers, fail_and_fail_requirement, expected_failure, Fail reason: The failure
tc, #.#, expect_helpers, fail_but_pass, failed, Test case was expecting a failure but none were raised
tc, #.#, expect_helpers, pass_and_pass, passed
tc, #.#, expect_helpers, pass_but_fail_check, failed, 1 checks failed; see output for more details
tc, #.#, expect_helpers, pass_but_fail_requirement, failed, Some reason
tc, #.#, expect_helpers, signal_any_and_signal, expected_signal, Call will signal
tc, #.#, expect_helpers, signal_but_pass, failed, Test case was expected to receive a termination signal but it continued execution
tc, #.#, expect_helpers, signal_no_and_signal, expected_signal, Call will signal
tc, #.#, expect_helpers, timeout_and_hang, expected_timeout, Will overrun
tc, #.#, expect_helpers, timeout_but_pass, failed, Test case was expected to hang but it continued execution
tp, #.#, expect_helpers, failed
EOF
# NO_CHECK_STYLE_END
    atf_check -s eq:0 -o file:expout -e empty -x \
        "atf-report -o csv:- <tps.out | " \
        "sed -E -e 's/[0-9]+.[0-9]{6}, /#.#, /'"

# NO_CHECK_STYLE_BEGIN
    cat >expout <<EOF
expect_helpers (1/1): 17 test cases
    death_and_exit: [#.#s] Expected failure: Exit case
    death_and_signal: [#.#s] Expected failure: Signal case
    death_but_pass: [#.#s] Failed: Test case was expected to terminate abruptly but it continued execution
    exit_any_and_exit: [#.#s] Expected failure: Call will exit
    exit_but_pass: [#.#s] Failed: Test case was expected to exit cleanly but it continued execution
    exit_code_and_exit: [#.#s] Expected failure: Call will exit
    fail_and_fail_check: [#.#s] Expected failure: And fail again: 2 checks failed as expected; see output for more details
    fail_and_fail_requirement: [#.#s] Expected failure: Fail reason: The failure
    fail_but_pass: [#.#s] Failed: Test case was expecting a failure but none were raised
    pass_and_pass: [#.#s] Passed.
    pass_but_fail_check: [#.#s] Failed: 1 checks failed; see output for more details
    pass_but_fail_requirement: [#.#s] Failed: Some reason
    signal_any_and_signal: [#.#s] Expected failure: Call will signal
    signal_but_pass: [#.#s] Failed: Test case was expected to receive a termination signal but it continued execution
    signal_no_and_signal: [#.#s] Expected failure: Call will signal
    timeout_and_hang: [#.#s] Expected failure: Will overrun
    timeout_but_pass: [#.#s] Failed: Test case was expected to hang but it continued execution
[#.#s]

Test cases for known bugs:
    expect_helpers:death_and_exit: Exit case
    expect_helpers:death_and_signal: Signal case
    expect_helpers:exit_any_and_exit: Call will exit
    expect_helpers:exit_code_and_exit: Call will exit
    expect_helpers:fail_and_fail_check: And fail again: 2 checks failed as expected; see output for more details
    expect_helpers:fail_and_fail_requirement: Fail reason: The failure
    expect_helpers:signal_any_and_signal: Call will signal
    expect_helpers:signal_no_and_signal: Call will signal
    expect_helpers:timeout_and_hang: Will overrun

Failed test cases:
    expect_helpers:death_but_pass, expect_helpers:exit_but_pass, expect_helpers:fail_but_pass, expect_helpers:pass_but_fail_check, expect_helpers:pass_but_fail_requirement, expect_helpers:signal_but_pass, expect_helpers:timeout_but_pass

Summary for 1 test programs:
    1 passed test cases.
    7 failed test cases.
    9 expected failed test cases.
    0 skipped test cases.
EOF
# NO_CHECK_STYLE_END
    atf_check -s eq:0 -o file:expout -e empty -x \
        "atf-report -o ticker:- <tps.out | " \
        "sed -E -e 's/[0-9]+.[0-9]{6}/#.#/'"

    # Just ensure that this does not crash for now...
    atf_check -s eq:0 -o ignore -e empty -x "atf-report -o xml:- <tps.out"
}

atf_test_case oflag
oflag_head()
{
    atf_set "descr" "Checks that the -o flag works"
}
oflag_body()
{
    create_helpers
    run_helpers

    # Get the default output.
    atf_check -s eq:0 -o save:stdout -e empty -x 'atf-report <tps.out'
    mv stdout defout

    # Check that changing the stdout output works.
    atf_check -s eq:0 -o save:stdout -e empty -x 'atf-report -o csv:- <tps.out'
    atf_check -s eq:1 -o empty -e empty cmp -s defout stdout
    cp stdout expcsv

    # Check that sending the output to a file does not write to stdout.
    atf_check -s eq:0 -o empty -e empty -x 'atf-report -o csv:fmt.out <tps.out'
    atf_check -s eq:0 -o empty -e empty cmp -s expcsv fmt.out
    rm -f fmt.out

    # Check that defining two outputs using the same format works.
    atf_check -s eq:0 -o empty -e empty -x \
              'atf-report -o csv:fmt.out -o csv:fmt2.out <tps.out'
    atf_check -s eq:0 -o empty -e empty cmp -s expcsv fmt.out
    atf_check -s eq:0 -o empty -e empty cmp -s fmt.out fmt2.out
    rm -f fmt.out fmt2.out

    # Check that defining two outputs using different formats works.
    atf_check -s eq:0 -o empty -e empty -x \
              'atf-report -o csv:fmt.out -o ticker:fmt2.out <tps.out'
    atf_check -s eq:0 -o empty -e empty cmp -s expcsv fmt.out
    atf_check -s eq:1 -o empty -e empty cmp -s fmt.out fmt2.out
    atf_check -s eq:0 -o ignore -e empty grep "test cases" fmt2.out
    atf_check -s eq:0 -o ignore -e empty grep "Failed test cases" fmt2.out
    atf_check -s eq:0 -o ignore -e empty grep "Summary for" fmt2.out
    rm -f fmt.out fmt2.out

    # Check that defining two outputs over the same file does not work.
    atf_check -s eq:1 -o empty -e match:'more than once' -x \
              'atf-report -o csv:fmt.out -o ticker:fmt.out <tps.out'
    rm -f fmt.out

    # Check that defining two outputs over stdout (but using different
    # paths) does not work.
    atf_check -s eq:1 -o empty -e match:'more than once' -x \
              'atf-report -o csv:- -o ticker:/dev/stdout <tps.out'
    rm -f fmt.out
}

atf_test_case output_csv
output_csv_head()
{
    atf_set "descr" "Checks the CSV output format"
}
output_csv_body()
{
    create_helpers
    run_helpers

# NO_CHECK_STYLE_BEGIN
    cat >expout <<EOF
tc, #.#, dir1/tp1, main, passed
tp, #.#, dir1/tp1, passed
tc, #.#, dir1/tp2, main, failed, This always fails
tp, #.#, dir1/tp2, failed
tc, #.#, tp3, main, passed
tp, #.#, tp3, passed
tc, #.#, tp4, main, failed, This always fails
tp, #.#, tp4, failed
tp, #.#, tp5, bogus, Invalid format for test case list: 1: Unexpected token \`<<NEWLINE>>'; expected \`:'
EOF
# NO_CHECK_STYLE_END

    atf_check -s eq:0 -o file:expout -e empty -x \
        "atf-report -o csv:- <tps.out | sed -E -e 's/[0-9]+.[0-9]{6}, /#.#, /'"
}

atf_test_case output_ticker
output_ticker_head()
{
    atf_set "descr" "Checks the ticker output format"
}
output_ticker_body()
{
    create_helpers
    run_helpers

# NO_CHECK_STYLE_BEGIN
    cat >expout <<EOF
dir1/tp1 (1/5): 1 test cases
    main: [#.#s] Passed.
[#.#s]

dir1/tp2 (2/5): 1 test cases
    main: [#.#s] Failed: This always fails
[#.#s]

tp3 (3/5): 1 test cases
    main: [#.#s] Passed.
[#.#s]

tp4 (4/5): 1 test cases
    main: [#.#s] Failed: This always fails
[#.#s]

tp5 (5/5): 0 test cases
tp5: BOGUS TEST PROGRAM: Cannot trust its results because of \`Invalid format for test case list: 1: Unexpected token \`<<NEWLINE>>'; expected \`:''
[#.#s]

Failed (bogus) test programs:
    tp5

Failed test cases:
    dir1/tp2:main, tp4:main

Summary for 5 test programs:
    2 passed test cases.
    2 failed test cases.
    0 expected failed test cases.
    0 skipped test cases.
EOF

    atf_check -s eq:0 -o file:expout -e empty -x \
        "atf-report -o ticker:- <tps.out | sed -E -e 's/[0-9]+.[0-9]{6}/#.#/'"
}
# NO_CHECK_STYLE_END

atf_test_case output_xml
output_xml_head()
{
    atf_set "descr" "Checks the XML output format"
}
output_xml_body()
{
    create_helpers
    run_helpers

# NO_CHECK_STYLE_BEGIN
    cat >expout <<EOF
<?xml version="1.0" encoding="ISO-8859-1"?>
<!DOCTYPE tests-results PUBLIC "-//NetBSD//DTD ATF Tests Results 0.1//EN" "http://www.NetBSD.org/XML/atf/tests-results.dtd">

<tests-results>
<info class="startinfo">A value</info>
<tp id="dir1/tp1">
<tc id="main">
<passed />
<tc-time>#.#</tc-time></tc>
<tp-time>#.#</tp-time></tp>
<tp id="dir1/tp2">
<tc id="main">
<failed>This always fails</failed>
<tc-time>#.#</tc-time></tc>
<tp-time>#.#</tp-time></tp>
<tp id="tp3">
<tc id="main">
<passed />
<tc-time>#.#</tc-time></tc>
<tp-time>#.#</tp-time></tp>
<tp id="tp4">
<tc id="main">
<failed>This always fails</failed>
<tc-time>#.#</tc-time></tc>
<tp-time>#.#</tp-time></tp>
<tp id="tp5">
<failed>Invalid format for test case list: 1: Unexpected token \`&lt;&lt;NEWLINE&gt;&gt;'; expected \`:'</failed>
<tp-time>#.#</tp-time></tp>
<info class="endinfo">Another value</info>
</tests-results>
EOF
# NO_CHECK_STYLE_END

    atf_check -s eq:0 -o file:expout -e empty -x \
        "atf-report -o xml:- < tps.out | sed -E -e 's/>[0-9]+.[0-9]{6}</>#.#</'"
}

atf_test_case output_xml_space
output_xml_space_head()
{
    atf_set "descr" "Checks that the XML output format properly preserves" \
                    "leading and trailing whitespace in stdout and stderr" \
                    "lines"
}
output_xml_space_body()
{
    export TESTCASE=diff
    cp $(atf_get_srcdir)/misc_helpers .
    cat >Atffile <<EOF
Content-Type: application/X-atf-atffile; version="1"

prop: test-suite = atf

tp: misc_helpers
EOF

# NO_CHECK_STYLE_BEGIN
    cat >expout <<EOF
<?xml version="1.0" encoding="ISO-8859-1"?>
<!DOCTYPE tests-results PUBLIC "-//NetBSD//DTD ATF Tests Results 0.1//EN" "http://www.NetBSD.org/XML/atf/tests-results.dtd">

<tests-results>
<info class="startinfo">A value</info>
<tp id="misc_helpers">
<tc id="diff">
<so>--- a	2007-11-04 14:00:41.000000000 +0100</so>
<so>+++ b	2007-11-04 14:00:48.000000000 +0100</so>
<so>@@ -1,7 +1,7 @@</so>
<so> This test is meant to simulate a diff.</so>
<so> Blank space at beginning of context lines must be preserved.</so>
<so> </so>
<so>-First original line.</so>
<so>-Second original line.</so>
<so>+First modified line.</so>
<so>+Second modified line.</so>
<so> </so>
<so> EOF</so>
<passed />
<tc-time>#.#</tc-time></tc>
<tp-time>#.#</tp-time></tp>
<info class="endinfo">Another value</info>
</tests-results>
EOF
# NO_CHECK_STYLE_END

    run_helpers
    atf_check -s eq:0 -o file:expout -e empty -x \
        "atf-report -o xml:- <tps.out | sed -E -e 's/>[0-9]+.[0-9]{6}</>#.#</'"
}

atf_test_case too_many_args
too_many_args_body()
{
    cat >experr <<EOF
atf-report: ERROR: No arguments allowed
EOF
    atf_check -s eq:1 -o empty -e file:experr atf-report foo
}

atf_init_test_cases()
{
    atf_add_test_case default
    atf_add_test_case expect
    atf_add_test_case oflag
    atf_add_test_case output_csv
    atf_add_test_case output_ticker
    atf_add_test_case output_xml
    atf_add_test_case output_xml_space
    atf_add_test_case too_many_args
}

# vim: syntax=sh:expandtab:shiftwidth=4:softtabstop=4