1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
|
# needs-profiler-support
# ignore-windows-gnu
# FIXME(pietroalbini): this test currently does not work on cross-compiled
# targets because remote-test is not capable of sending back the *.profraw
# files generated by the LLVM instrumentation.
# ignore-cross-compile
# Rust coverage maps support LLVM Coverage Mapping Format versions 5 and 6,
# corresponding with LLVM versions 12 and 13, respectively.
# When upgrading LLVM versions, consider whether to enforce a minimum LLVM
# version during testing, with an additional directive at the top of this file
# that sets, for example: `min-llvm-version: 12.0`
# FIXME(mati865): MinGW GCC miscompiles compiler-rt profiling library but with Clang it works
# properly. Since we only have GCC on the CI ignore the test for now.
include ../coverage/coverage_tools.mk
BASEDIR=../coverage-reports
SOURCEDIR=../coverage
# The `llvm-cov show` flag `--debug`, used to generate the `counters` output files, is only
# enabled if LLVM assertions are enabled. This requires Rust config `llvm/optimize` and not
# `llvm/release_debuginfo`. Note that some CI builds disable debug assertions (by setting
# `NO_LLVM_ASSERTIONS=1`), so the tests must still pass even if the `--debug` flag is
# not supported. (Note that `counters` files are only produced in the `$(TMPDIR)`
# directory, for inspection and debugging support. They are *not* copied to `expected_*`
# files when `--bless`ed.)
LLVM_COV_DEBUG := $(shell \
"$(LLVM_BIN_DIR)"/llvm-cov show --debug 2>&1 | \
grep -q "Unknown command line argument '--debug'"; \
echo $$?)
ifeq ($(LLVM_COV_DEBUG), 1)
DEBUG_FLAG=--debug
endif
# FIXME(richkadel): I'm adding `--ignore-filename-regex=` line(s) for specific test(s) that produce
# `llvm-cov` results for multiple files (for example `uses_crate.rs` and `used_crate/mod.rs`) as a
# workaround for two problems causing tests to fail on Windows:
#
# 1. When multiple files appear in the `llvm-cov show` results, each file's coverage results can
# appear in different a different order. Whether this is random or, somehow, platform-specific,
# the Windows output flips the order of the files, compared to Linux. In the `uses_crate.rs`
# test, the only test-unique (interesting) results we care about are the results for only one
# of the two files, `mod/uses_crate.rs`, so the workaround is to ignore all but this one file.
# In the future, we may want a more sophisticated solution that splits apart `llvm-cov show`
# results into separate results files for each result (taking care not to create new file
# paths that might be too long for Windows MAX_PATH limits when creating these new sub-results,
# as well).
# 2. When multiple files appear in the `llvm-cov show` results, the results for each file are
# prefixed with their filename, including platform-specific path separators (`\` for Windows,
# and `/` everywhere else). This could be filtered or normalized of course, but by ignoring
# coverage results for all but one of the file, the filenames are no longer included anyway.
# If this changes (if/when we decide to support `llvm-cov show` results for multiple files),
# the file path separator differences may need to be addressed.
#
# Since this is only a workaround, I decided to implement the override by adding an option for
# each file to be ignored, using a `--ignore-filename-regex=` entry for each one, rather than
# implement some more sophisticated solution with a new custom test directive in the test file
# itself (similar to `expect-exit-status`) because that would add a lot of complexity and still
# be a workaround, with the same result, with no benefit.
#
# Yes these `--ignore-filename-regex=` options are included in all invocations of `llvm-cov show`
# for now, but it is effectively ignored for all tests that don't include this file anyway.
#
# (Note that it's also possible the `_counters.<test>.txt` and `<test>.json` files (if generated)
# may order results from multiple files inconsistently, which might also have to be accommodated
# if and when we allow `llvm-cov` to produce results for multiple files. Note, the path separators
# appear to be normalized to `/` in those files, thankfully.)
LLVM_COV_IGNORE_FILES=\
--ignore-filename-regex='(uses_crate.rs|uses_inline_crate.rs|unused_mod.rs)'
all: $(patsubst $(SOURCEDIR)/lib/%.rs,%,$(wildcard $(SOURCEDIR)/lib/*.rs)) $(patsubst $(SOURCEDIR)/%.rs,%,$(wildcard $(SOURCEDIR)/*.rs))
# Ensure there are no `expected` results for tests that may have been removed or renamed
.PHONY: clear_expected_if_blessed
clear_expected_if_blessed:
ifdef RUSTC_BLESS_TEST
rm -f expected_*
endif
-include clear_expected_if_blessed
%: $(SOURCEDIR)/lib/%.rs
# Compile the test library with coverage instrumentation
$(RUSTC) $(SOURCEDIR)/lib/$@.rs \
$$( sed -n 's/^\/\/ compile-flags: \([^#]*\).*/\1/p' $(SOURCEDIR)/lib/$@.rs ) \
--crate-type rlib -Cinstrument-coverage --target $(TARGET)
%: $(SOURCEDIR)/%.rs
# Compile the test program with coverage instrumentation
$(RUSTC) $(SOURCEDIR)/$@.rs \
$$( sed -n 's/^\/\/ compile-flags: \([^#]*\).*/\1/p' $(SOURCEDIR)/$@.rs ) \
-L "$(TMPDIR)" -Cinstrument-coverage --target $(TARGET)
# Run it in order to generate some profiling data,
# with `LLVM_PROFILE_FILE=<profdata_file>` environment variable set to
# output the coverage stats for this run.
LLVM_PROFILE_FILE="$(TMPDIR)"/$@.profraw \
$(call RUN,$@) || \
( \
status=$$?; \
grep -q "^\/\/ expect-exit-status-$$status" $(SOURCEDIR)/$@.rs || \
( >&2 echo "program exited with an unexpected exit status: $$status"; \
false \
) \
)
# Run it through rustdoc as well to cover doctests.
# `%p` is the pid, and `%m` the binary signature. We suspect that the pid alone
# might result in overwritten files and failed tests, as rustdoc spawns each
# doctest as its own process, so make sure the filename is as unique as possible.
LLVM_PROFILE_FILE="$(TMPDIR)"/$@-%p-%m.profraw \
$(RUSTDOC) --crate-name workaround_for_79771 --test $(SOURCEDIR)/$@.rs \
$$( sed -n 's/^\/\/ compile-flags: \([^#]*\).*/\1/p' $(SOURCEDIR)/$@.rs ) \
-L "$(TMPDIR)" -Cinstrument-coverage \
-Z unstable-options --persist-doctests=$(TMPDIR)/rustdoc-$@
# Postprocess the profiling data so it can be used by the llvm-cov tool
"$(LLVM_BIN_DIR)"/llvm-profdata merge --sparse \
"$(TMPDIR)"/$@*.profraw \
-o "$(TMPDIR)"/$@.profdata
# Generate a coverage report using `llvm-cov show`.
"$(LLVM_BIN_DIR)"/llvm-cov show \
$(DEBUG_FLAG) \
$(LLVM_COV_IGNORE_FILES) \
--compilation-dir=. \
--Xdemangler="$(RUST_DEMANGLER)" \
--show-line-counts-or-regions \
--instr-profile="$(TMPDIR)"/$@.profdata \
$(call BIN,"$(TMPDIR)"/$@) \
$$( \
for file in $(TMPDIR)/rustdoc-$@/*/rust_out*; do \
[ -x "$$file" ] && printf "%s %s " -object $$file; \
done \
) \
2> "$(TMPDIR)"/show_coverage_stderr.$@.txt \
| "$(PYTHON)" $(BASEDIR)/normalize_paths.py \
| "$(PYTHON)" $(BASEDIR)/sort_subviews.py \
> "$(TMPDIR)"/actual_show_coverage.$@.txt || \
( status=$$? ; \
>&2 cat "$(TMPDIR)"/show_coverage_stderr.$@.txt ; \
exit $$status \
)
ifdef DEBUG_FLAG
# The first line (beginning with "Args:" contains hard-coded, build-specific
# file paths. Strip that line and keep the remaining lines with counter debug
# data.
tail -n +2 "$(TMPDIR)"/show_coverage_stderr.$@.txt \
> "$(TMPDIR)"/actual_show_coverage_counters.$@.txt
endif
ifdef RUSTC_BLESS_TEST
cp "$(TMPDIR)"/actual_show_coverage.$@.txt \
expected_show_coverage.$@.txt
else
# Compare the show coverage output (`--bless` refreshes `typical` files).
#
# `llvm-cov show` normally prints instantiation groups in an unpredictable
# order, but we have used `sort_subviews.py` to sort them, so we can still
# check the output directly with `diff`.
#
# Some of the test cases are currently not working (since #110393) and have
# been marked with `// ignore-llvm-cov-show-diffs` so that they don't fail
# the build.
$(DIFF) \
expected_show_coverage.$@.txt "$(TMPDIR)"/actual_show_coverage.$@.txt || \
( grep -q '^\/\/ ignore-llvm-cov-show-diffs' $(SOURCEDIR)/$@.rs && \
>&2 echo 'diff failed, but suppressed with `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs' \
) || \
( >&2 echo 'diff failed, and not suppressed without `// ignore-llvm-cov-show-diffs` in $(SOURCEDIR)/$@.rs'; \
false \
)
endif
|
