summaryrefslogtreecommitdiffstats
path: root/src/tools/linkchecker
diff options
context:
space:
mode:
Diffstat (limited to 'src/tools/linkchecker')
-rw-r--r--src/tools/linkchecker/Cargo.toml12
-rwxr-xr-xsrc/tools/linkchecker/linkcheck.sh116
-rw-r--r--src/tools/linkchecker/main.rs560
-rw-r--r--src/tools/linkchecker/tests/basic_broken/foo.html5
-rw-r--r--src/tools/linkchecker/tests/broken_fragment_local/foo.html5
-rw-r--r--src/tools/linkchecker/tests/broken_fragment_remote/bar.html4
-rw-r--r--src/tools/linkchecker/tests/broken_fragment_remote/inner/foo.html5
-rw-r--r--src/tools/linkchecker/tests/broken_redir/foo.html5
-rw-r--r--src/tools/linkchecker/tests/broken_redir/redir-bad.html11
-rw-r--r--src/tools/linkchecker/tests/checks.rs113
-rw-r--r--src/tools/linkchecker/tests/directory_link/foo.html5
-rw-r--r--src/tools/linkchecker/tests/directory_link/somedir/index.html4
-rw-r--r--src/tools/linkchecker/tests/redirect_loop/foo.html5
-rw-r--r--src/tools/linkchecker/tests/redirect_loop/redir-bad.html11
-rw-r--r--src/tools/linkchecker/tests/valid/inner/bar.html7
-rw-r--r--src/tools/linkchecker/tests/valid/inner/foo.html14
-rw-r--r--src/tools/linkchecker/tests/valid/inner/redir-bad.html12
-rw-r--r--src/tools/linkchecker/tests/valid/inner/redir-target.html5
-rw-r--r--src/tools/linkchecker/tests/valid/inner/redir.html11
-rw-r--r--src/tools/linkchecker/tests/valid/outer.html5
20 files changed, 915 insertions, 0 deletions
diff --git a/src/tools/linkchecker/Cargo.toml b/src/tools/linkchecker/Cargo.toml
new file mode 100644
index 000000000..1d8f2f918
--- /dev/null
+++ b/src/tools/linkchecker/Cargo.toml
@@ -0,0 +1,12 @@
+[package]
+name = "linkchecker"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+name = "linkchecker"
+path = "main.rs"
+
+[dependencies]
+regex = "1"
+once_cell = "1"
diff --git a/src/tools/linkchecker/linkcheck.sh b/src/tools/linkchecker/linkcheck.sh
new file mode 100755
index 000000000..9eeebf444
--- /dev/null
+++ b/src/tools/linkchecker/linkcheck.sh
@@ -0,0 +1,116 @@
+#!/bin/sh
+#
+# This is a script that can be used in each book's CI to validate links using
+# the same tool as rust-lang/rust.
+#
+# This requires the rust-docs rustup component to be installed in the nightly
+# toolchain.
+#
+# Usage:
+# ./linkcheck.sh <name-of-book>
+#
+# Options:
+#
+# -i "Iterative" mode. The script will not clean up after it is done so
+# you can inspect the result, and re-run more quickly.
+#
+# --all Check all books. This can help make sure you don't break links
+# from other books into your book.
+
+set -e
+
+if [ ! -f book.toml ] && [ ! -f src/SUMMARY.md ]
+then
+ echo "Run command in root directory of the book."
+ exit 1
+fi
+
+html_dir="$(rustc +nightly --print sysroot)/share/doc/rust/html"
+
+if [ ! -d "$html_dir" ]
+then
+ echo "HTML docs are missing from sysroot: $html_dir"
+ echo "Make sure the nightly rust-docs rustup component is installed."
+ exit 1
+fi
+
+# Avoid failure caused by newer mdbook.
+export MDBOOK_OUTPUT__HTML__INPUT_404=""
+
+book_name=""
+# Iterative will avoid cleaning up, so you can quickly run it repeatedly.
+iterative=0
+# If "1", test all books, else only this book.
+all_books=0
+
+while [ "$1" != "" ]
+do
+ case "$1" in
+ -i)
+ iterative=1
+ ;;
+ --all)
+ all_books=1
+ ;;
+ *)
+ if [ -n "$book_name" ]
+ then
+ echo "only one argument allowed"
+ exit 1
+ fi
+ book_name="$1"
+ ;;
+ esac
+ shift
+done
+
+if [ -z "$book_name" ]
+then
+ echo "usage: $0 <name-of-book>"
+ exit 1
+fi
+
+if [ ! -d "$html_dir/$book_name" ]
+then
+ echo "book name \"$book_name\" not found in sysroot \"$html_dir\""
+ exit 1
+fi
+
+if [ "$iterative" = "0" ]
+then
+ echo "Cleaning old directories..."
+ rm -rf linkcheck linkchecker
+fi
+
+if [ ! -e "linkchecker/main.rs" ] || [ "$iterative" = "0" ]
+then
+ echo "Downloading linkchecker source..."
+ nightly_hash=$(rustc +nightly -Vv | grep commit-hash | cut -f2 -d" ")
+ url="https://raw.githubusercontent.com/rust-lang/rust"
+ mkdir linkchecker
+ curl -o linkchecker/Cargo.toml ${url}/${nightly_hash}/src/tools/linkchecker/Cargo.toml
+ curl -o linkchecker/main.rs ${url}/${nightly_hash}/src/tools/linkchecker/main.rs
+fi
+
+echo "Building book \"$book_name\"..."
+mdbook build
+
+cp -R "$html_dir" linkcheck
+rm -rf "linkcheck/$book_name"
+cp -R book "linkcheck/$book_name"
+
+if [ "$all_books" = "1" ]
+then
+ check_path="linkcheck"
+else
+ check_path="linkcheck/$book_name"
+fi
+echo "Running linkchecker on \"$check_path\"..."
+cargo run --release --manifest-path=linkchecker/Cargo.toml -- "$check_path"
+
+if [ "$iterative" = "0" ]
+then
+ rm -rf linkcheck linkchecker
+fi
+
+echo "Link check completed successfully!"
diff --git a/src/tools/linkchecker/main.rs b/src/tools/linkchecker/main.rs
new file mode 100644
index 000000000..a7c78d80c
--- /dev/null
+++ b/src/tools/linkchecker/main.rs
@@ -0,0 +1,560 @@
+//! Script to check the validity of `href` links in our HTML documentation.
+//!
+//! In the past we've been quite error prone to writing in broken links as most
+//! of them are manually rather than automatically added. As files move over
+//! time or apis change old links become stale or broken. The purpose of this
+//! script is to check all relative links in our documentation to make sure they
+//! actually point to a valid place.
+//!
+//! Currently this doesn't actually do any HTML parsing or anything fancy like
+//! that, it just has a simple "regex" to search for `href` and `id` tags.
+//! These values are then translated to file URLs if possible and then the
+//! destination is asserted to exist.
+//!
+//! A few exceptions are allowed as there's known bugs in rustdoc, but this
+//! should catch the majority of "broken link" cases.
+
+use std::cell::RefCell;
+use std::collections::{HashMap, HashSet};
+use std::env;
+use std::fs;
+use std::io::ErrorKind;
+use std::path::{Component, Path, PathBuf};
+use std::rc::Rc;
+use std::time::Instant;
+
+use once_cell::sync::Lazy;
+use regex::Regex;
+
+// Add linkcheck exceptions here
+// If at all possible you should use intra-doc links to avoid linkcheck issues. These
+// are cases where that does not work
+// [(generated_documentation_page, &[broken_links])]
+#[rustfmt::skip]
+const LINKCHECK_EXCEPTIONS: &[(&str, &[&str])] = &[
+ // These try to link to std::collections, but are defined in alloc
+ // https://github.com/rust-lang/rust/issues/74481
+ ("std/collections/btree_map/struct.BTreeMap.html", &["#insert-and-complex-keys"]),
+ ("std/collections/btree_set/struct.BTreeSet.html", &["#insert-and-complex-keys"]),
+ ("alloc/collections/btree_map/struct.BTreeMap.html", &["#insert-and-complex-keys"]),
+ ("alloc/collections/btree_set/struct.BTreeSet.html", &["#insert-and-complex-keys"]),
+
+ // These try to link to various things in std, but are defined in core.
+ // The docs in std::primitive use proper intra-doc links, so these seem fine to special-case.
+ // Most these are broken because liballoc uses `#[lang_item]` magic to define things on
+ // primitives that aren't available in core.
+ ("alloc/slice/trait.Join.html", &["#method.join"]),
+ ("alloc/slice/trait.Concat.html", &["#method.concat"]),
+ ("alloc/slice/index.html", &["#method.concat", "#method.join"]),
+ ("alloc/vec/struct.Vec.html", &["#method.sort_by_key", "#method.sort_by_cached_key"]),
+ ("core/primitive.str.html", &["#method.to_ascii_uppercase", "#method.to_ascii_lowercase"]),
+ ("core/primitive.slice.html", &["#method.to_ascii_uppercase", "#method.to_ascii_lowercase",
+ "core/slice::sort_by_key", "core\\slice::sort_by_key",
+ "#method.sort_by_cached_key"]),
+];
+
+#[rustfmt::skip]
+const INTRA_DOC_LINK_EXCEPTIONS: &[(&str, &[&str])] = &[
+ // This will never have links that are not in other pages.
+ // To avoid repeating the exceptions twice, an empty list means all broken links are allowed.
+ ("reference/print.html", &[]),
+ // All the reference 'links' are actually ENBF highlighted as code
+ ("reference/comments.html", &[
+ "/</code> <code>!",
+ "*</code> <code>!",
+ ]),
+ ("reference/identifiers.html", &[
+ "a</code>-<code>z</code> <code>A</code>-<code>Z",
+ "a</code>-<code>z</code> <code>A</code>-<code>Z</code> <code>0</code>-<code>9</code> <code>_",
+ "a</code>-<code>z</code> <code>A</code>-<code>Z</code>] [<code>a</code>-<code>z</code> <code>A</code>-<code>Z</code> <code>0</code>-<code>9</code> <code>_",
+ ]),
+ ("reference/tokens.html", &[
+ "0</code>-<code>1",
+ "0</code>-<code>7",
+ "0</code>-<code>9",
+ "0</code>-<code>9",
+ "0</code>-<code>9</code> <code>a</code>-<code>f</code> <code>A</code>-<code>F",
+ ]),
+ ("reference/notation.html", &[
+ "b</code> <code>B",
+ "a</code>-<code>z",
+ ]),
+ // This is being used in the sense of 'inclusive range', not a markdown link
+ ("core/ops/struct.RangeInclusive.html", &["begin</code>, <code>end"]),
+ ("std/ops/struct.RangeInclusive.html", &["begin</code>, <code>end"]),
+ ("core/slice/trait.SliceIndex.html", &["begin</code>, <code>end"]),
+ ("alloc/slice/trait.SliceIndex.html", &["begin</code>, <code>end"]),
+ ("std/slice/trait.SliceIndex.html", &["begin</code>, <code>end"]),
+ ("core/primitive.str.html", &["begin</code>, <code>end"]),
+ ("std/primitive.str.html", &["begin</code>, <code>end"]),
+
+];
+
+static BROKEN_INTRA_DOC_LINK: Lazy<Regex> =
+ Lazy::new(|| Regex::new(r#"\[<code>(.*)</code>\]"#).unwrap());
+
+macro_rules! t {
+ ($e:expr) => {
+ match $e {
+ Ok(e) => e,
+ Err(e) => panic!("{} failed with {:?}", stringify!($e), e),
+ }
+ };
+}
+
+fn main() {
+ let docs = env::args_os().nth(1).expect("doc path should be first argument");
+ let docs = env::current_dir().unwrap().join(docs);
+ let mut checker = Checker { root: docs.clone(), cache: HashMap::new() };
+ let mut report = Report {
+ errors: 0,
+ start: Instant::now(),
+ html_files: 0,
+ html_redirects: 0,
+ links_checked: 0,
+ links_ignored_external: 0,
+ links_ignored_exception: 0,
+ intra_doc_exceptions: 0,
+ };
+ checker.walk(&docs, &mut report);
+ report.report();
+ if report.errors != 0 {
+ println!("found some broken links");
+ std::process::exit(1);
+ }
+}
+
+struct Checker {
+ root: PathBuf,
+ cache: Cache,
+}
+
+struct Report {
+ errors: u32,
+ start: Instant,
+ html_files: u32,
+ html_redirects: u32,
+ links_checked: u32,
+ links_ignored_external: u32,
+ links_ignored_exception: u32,
+ intra_doc_exceptions: u32,
+}
+
+/// A cache entry.
+enum FileEntry {
+ /// An HTML file.
+ ///
+ /// This includes the contents of the HTML file, and an optional set of
+ /// HTML IDs. The IDs are used for checking fragments. They are computed
+ /// as-needed. The source is discarded (replaced with an empty string)
+ /// after the file has been checked, to conserve on memory.
+ HtmlFile { source: Rc<String>, ids: RefCell<HashSet<String>> },
+ /// This file is an HTML redirect to the given local path.
+ Redirect { target: PathBuf },
+ /// This is not an HTML file.
+ OtherFile,
+ /// This is a directory.
+ Dir,
+ /// The file doesn't exist.
+ Missing,
+}
+
+/// A cache to speed up file access.
+type Cache = HashMap<String, FileEntry>;
+
+fn small_url_encode(s: &str) -> String {
+ s.replace("<", "%3C")
+ .replace(">", "%3E")
+ .replace(" ", "%20")
+ .replace("?", "%3F")
+ .replace("'", "%27")
+ .replace("&", "%26")
+ .replace(",", "%2C")
+ .replace(":", "%3A")
+ .replace(";", "%3B")
+ .replace("[", "%5B")
+ .replace("]", "%5D")
+ .replace("\"", "%22")
+}
+
+impl Checker {
+ /// Primary entry point for walking the filesystem to find HTML files to check.
+ fn walk(&mut self, dir: &Path, report: &mut Report) {
+ for entry in t!(dir.read_dir()).map(|e| t!(e)) {
+ let path = entry.path();
+ // Goes through symlinks
+ let metadata = t!(fs::metadata(&path));
+ if metadata.is_dir() {
+ self.walk(&path, report);
+ } else {
+ self.check(&path, report);
+ }
+ }
+ }
+
+ /// Checks a single file.
+ fn check(&mut self, file: &Path, report: &mut Report) {
+ let (pretty_path, entry) = self.load_file(file, report);
+ let source = match entry {
+ FileEntry::Missing => panic!("missing file {:?} while walking", file),
+ FileEntry::Dir => unreachable!("never with `check` path"),
+ FileEntry::OtherFile => return,
+ FileEntry::Redirect { .. } => return,
+ FileEntry::HtmlFile { source, ids } => {
+ parse_ids(&mut ids.borrow_mut(), &pretty_path, source, report);
+ source.clone()
+ }
+ };
+
+ // Search for anything that's the regex 'href[ ]*=[ ]*".*?"'
+ with_attrs_in_source(&source, " href", |url, i, base| {
+ // Ignore external URLs
+ if url.starts_with("http:")
+ || url.starts_with("https:")
+ || url.starts_with("javascript:")
+ || url.starts_with("ftp:")
+ || url.starts_with("irc:")
+ || url.starts_with("data:")
+ {
+ report.links_ignored_external += 1;
+ return;
+ }
+ report.links_checked += 1;
+ let (url, fragment) = match url.split_once('#') {
+ None => (url, None),
+ Some((url, fragment)) => (url, Some(fragment)),
+ };
+ // NB: the `splitn` always succeeds, even if the delimiter is not present.
+ let url = url.splitn(2, '?').next().unwrap();
+
+ // Once we've plucked out the URL, parse it using our base url and
+ // then try to extract a file path.
+ let mut path = file.to_path_buf();
+ if !base.is_empty() || !url.is_empty() {
+ path.pop();
+ for part in Path::new(base).join(url).components() {
+ match part {
+ Component::Prefix(_) | Component::RootDir => {
+ // Avoid absolute paths as they make the docs not
+ // relocatable by making assumptions on where the docs
+ // are hosted relative to the site root.
+ report.errors += 1;
+ println!(
+ "{}:{}: absolute path - {}",
+ pretty_path,
+ i + 1,
+ Path::new(base).join(url).display()
+ );
+ return;
+ }
+ Component::CurDir => {}
+ Component::ParentDir => {
+ path.pop();
+ }
+ Component::Normal(s) => {
+ path.push(s);
+ }
+ }
+ }
+ }
+
+ let (target_pretty_path, target_entry) = self.load_file(&path, report);
+ let (target_source, target_ids) = match target_entry {
+ FileEntry::Missing => {
+ if is_exception(file, &target_pretty_path) {
+ report.links_ignored_exception += 1;
+ } else {
+ report.errors += 1;
+ println!(
+ "{}:{}: broken link - `{}`",
+ pretty_path,
+ i + 1,
+ target_pretty_path
+ );
+ }
+ return;
+ }
+ FileEntry::Dir => {
+ // Links to directories show as directory listings when viewing
+ // the docs offline so it's best to avoid them.
+ report.errors += 1;
+ println!(
+ "{}:{}: directory link to `{}` \
+ (directory links should use index.html instead)",
+ pretty_path,
+ i + 1,
+ target_pretty_path
+ );
+ return;
+ }
+ FileEntry::OtherFile => return,
+ FileEntry::Redirect { target } => {
+ let t = target.clone();
+ drop(target);
+ let (target, redir_entry) = self.load_file(&t, report);
+ match redir_entry {
+ FileEntry::Missing => {
+ report.errors += 1;
+ println!(
+ "{}:{}: broken redirect from `{}` to `{}`",
+ pretty_path,
+ i + 1,
+ target_pretty_path,
+ target
+ );
+ return;
+ }
+ FileEntry::Redirect { target } => {
+ // Redirect to a redirect, this link checker
+ // currently doesn't support this, since it would
+ // require cycle checking, etc.
+ report.errors += 1;
+ println!(
+ "{}:{}: redirect from `{}` to `{}` \
+ which is also a redirect (not supported)",
+ pretty_path,
+ i + 1,
+ target_pretty_path,
+ target.display()
+ );
+ return;
+ }
+ FileEntry::Dir => {
+ report.errors += 1;
+ println!(
+ "{}:{}: redirect from `{}` to `{}` \
+ which is a directory \
+ (directory links should use index.html instead)",
+ pretty_path,
+ i + 1,
+ target_pretty_path,
+ target
+ );
+ return;
+ }
+ FileEntry::OtherFile => return,
+ FileEntry::HtmlFile { source, ids } => (source, ids),
+ }
+ }
+ FileEntry::HtmlFile { source, ids } => (source, ids),
+ };
+
+ // Alright, if we've found an HTML file for the target link. If
+ // this is a fragment link, also check that the `id` exists.
+ if let Some(ref fragment) = fragment {
+ // Fragments like `#1-6` are most likely line numbers to be
+ // interpreted by javascript, so we're ignoring these
+ if fragment.splitn(2, '-').all(|f| f.chars().all(|c| c.is_numeric())) {
+ return;
+ }
+
+ parse_ids(&mut target_ids.borrow_mut(), &pretty_path, target_source, report);
+
+ if target_ids.borrow().contains(*fragment) {
+ return;
+ }
+
+ if is_exception(file, &format!("#{}", fragment)) {
+ report.links_ignored_exception += 1;
+ } else {
+ report.errors += 1;
+ print!("{}:{}: broken link fragment ", pretty_path, i + 1);
+ println!("`#{}` pointing to `{}`", fragment, target_pretty_path);
+ };
+ }
+ });
+
+ // Search for intra-doc links that rustdoc didn't warn about
+ // FIXME(#77199, 77200) Rustdoc should just warn about these directly.
+ // NOTE: only looks at one line at a time; in practice this should find most links
+ for (i, line) in source.lines().enumerate() {
+ for broken_link in BROKEN_INTRA_DOC_LINK.captures_iter(line) {
+ if is_intra_doc_exception(file, &broken_link[1]) {
+ report.intra_doc_exceptions += 1;
+ } else {
+ report.errors += 1;
+ print!("{}:{}: broken intra-doc link - ", pretty_path, i + 1);
+ println!("{}", &broken_link[0]);
+ }
+ }
+ }
+ // we don't need the source anymore,
+ // so drop to reduce memory-usage
+ match self.cache.get_mut(&pretty_path).unwrap() {
+ FileEntry::HtmlFile { source, .. } => *source = Rc::new(String::new()),
+ _ => unreachable!("must be html file"),
+ }
+ }
+
+ /// Load a file from disk, or from the cache if available.
+ fn load_file(&mut self, file: &Path, report: &mut Report) -> (String, &FileEntry) {
+ // https://docs.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499-
+ #[cfg(windows)]
+ const ERROR_INVALID_NAME: i32 = 123;
+
+ let pretty_path =
+ file.strip_prefix(&self.root).unwrap_or(&file).to_str().unwrap().to_string();
+
+ let entry =
+ self.cache.entry(pretty_path.clone()).or_insert_with(|| match fs::metadata(file) {
+ Ok(metadata) if metadata.is_dir() => FileEntry::Dir,
+ Ok(_) => {
+ if file.extension().and_then(|s| s.to_str()) != Some("html") {
+ FileEntry::OtherFile
+ } else {
+ report.html_files += 1;
+ load_html_file(file, report)
+ }
+ }
+ Err(e) if e.kind() == ErrorKind::NotFound => FileEntry::Missing,
+ Err(e) => {
+ // If a broken intra-doc link contains `::`, on windows, it will cause `ERROR_INVALID_NAME` rather than `NotFound`.
+ // Explicitly check for that so that the broken link can be allowed in `LINKCHECK_EXCEPTIONS`.
+ #[cfg(windows)]
+ if e.raw_os_error() == Some(ERROR_INVALID_NAME)
+ && file.as_os_str().to_str().map_or(false, |s| s.contains("::"))
+ {
+ return FileEntry::Missing;
+ }
+ panic!("unexpected read error for {}: {}", file.display(), e);
+ }
+ });
+ (pretty_path, entry)
+ }
+}
+
+impl Report {
+ fn report(&self) {
+ println!("checked links in: {:.1}s", self.start.elapsed().as_secs_f64());
+ println!("number of HTML files scanned: {}", self.html_files);
+ println!("number of HTML redirects found: {}", self.html_redirects);
+ println!("number of links checked: {}", self.links_checked);
+ println!("number of links ignored due to external: {}", self.links_ignored_external);
+ println!("number of links ignored due to exceptions: {}", self.links_ignored_exception);
+ println!("number of intra doc links ignored: {}", self.intra_doc_exceptions);
+ println!("errors found: {}", self.errors);
+ }
+}
+
+fn load_html_file(file: &Path, report: &mut Report) -> FileEntry {
+ let source = match fs::read_to_string(file) {
+ Ok(s) => Rc::new(s),
+ Err(err) => {
+ // This usually should not fail since `metadata` was already
+ // called successfully on this file.
+ panic!("unexpected read error for {}: {}", file.display(), err);
+ }
+ };
+ match maybe_redirect(&source) {
+ Some(target) => {
+ report.html_redirects += 1;
+ let target = file.parent().unwrap().join(target);
+ FileEntry::Redirect { target }
+ }
+ None => FileEntry::HtmlFile { source: source.clone(), ids: RefCell::new(HashSet::new()) },
+ }
+}
+
+fn is_intra_doc_exception(file: &Path, link: &str) -> bool {
+ if let Some(entry) = INTRA_DOC_LINK_EXCEPTIONS.iter().find(|&(f, _)| file.ends_with(f)) {
+ entry.1.is_empty() || entry.1.contains(&link)
+ } else {
+ false
+ }
+}
+
+fn is_exception(file: &Path, link: &str) -> bool {
+ if let Some(entry) = LINKCHECK_EXCEPTIONS.iter().find(|&(f, _)| file.ends_with(f)) {
+ entry.1.contains(&link)
+ } else {
+ // FIXME(#63351): Concat trait in alloc/slice reexported in primitive page
+ //
+ // NOTE: This cannot be added to `LINKCHECK_EXCEPTIONS` because the resolved path
+ // calculated in `check` function is outside `build/<triple>/doc` dir.
+ // So the `strip_prefix` method just returns the old absolute broken path.
+ if file.ends_with("std/primitive.slice.html") {
+ if link.ends_with("primitive.slice.html") {
+ return true;
+ }
+ }
+ false
+ }
+}
+
+/// If the given HTML file contents is an HTML redirect, this returns the
+/// destination path given in the redirect.
+fn maybe_redirect(source: &str) -> Option<String> {
+ const REDIRECT_RUSTDOC: (usize, &str) = (7, "<p>Redirecting to <a href=");
+ const REDIRECT_MDBOOK: (usize, &str) = (8 - 7, "<p>Redirecting to... <a href=");
+
+ let mut lines = source.lines();
+
+ let mut find_redirect = |(line_rel, redirect_pattern): (usize, &str)| {
+ let redirect_line = lines.nth(line_rel)?;
+
+ redirect_line.find(redirect_pattern).map(|i| {
+ let rest = &redirect_line[(i + redirect_pattern.len() + 1)..];
+ let pos_quote = rest.find('"').unwrap();
+ rest[..pos_quote].to_owned()
+ })
+ };
+
+ find_redirect(REDIRECT_RUSTDOC).or_else(|| find_redirect(REDIRECT_MDBOOK))
+}
+
+fn with_attrs_in_source<F: FnMut(&str, usize, &str)>(source: &str, attr: &str, mut f: F) {
+ let mut base = "";
+ for (i, mut line) in source.lines().enumerate() {
+ while let Some(j) = line.find(attr) {
+ let rest = &line[j + attr.len()..];
+ // The base tag should always be the first link in the document so
+ // we can get away with using one pass.
+ let is_base = line[..j].ends_with("<base");
+ line = rest;
+ let pos_equals = match rest.find('=') {
+ Some(i) => i,
+ None => continue,
+ };
+ if rest[..pos_equals].trim_start_matches(' ') != "" {
+ continue;
+ }
+
+ let rest = &rest[pos_equals + 1..];
+
+ let pos_quote = match rest.find(&['"', '\''][..]) {
+ Some(i) => i,
+ None => continue,
+ };
+ let quote_delim = rest.as_bytes()[pos_quote] as char;
+
+ if rest[..pos_quote].trim_start_matches(' ') != "" {
+ continue;
+ }
+ let rest = &rest[pos_quote + 1..];
+ let url = match rest.find(quote_delim) {
+ Some(i) => &rest[..i],
+ None => continue,
+ };
+ if is_base {
+ base = url;
+ continue;
+ }
+ f(url, i, base)
+ }
+ }
+}
+
+fn parse_ids(ids: &mut HashSet<String>, file: &str, source: &str, report: &mut Report) {
+ if ids.is_empty() {
+ with_attrs_in_source(source, " id", |fragment, i, _| {
+ let frag = fragment.trim_start_matches("#").to_owned();
+ let encoded = small_url_encode(&frag);
+ if !ids.insert(frag) {
+ report.errors += 1;
+ println!("{}:{}: id is not unique: `{}`", file, i, fragment);
+ }
+ // Just in case, we also add the encoded id.
+ ids.insert(encoded);
+ });
+ }
+}
diff --git a/src/tools/linkchecker/tests/basic_broken/foo.html b/src/tools/linkchecker/tests/basic_broken/foo.html
new file mode 100644
index 000000000..cb27c55c9
--- /dev/null
+++ b/src/tools/linkchecker/tests/basic_broken/foo.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+<a href="bar.html">test</a>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/broken_fragment_local/foo.html b/src/tools/linkchecker/tests/broken_fragment_local/foo.html
new file mode 100644
index 000000000..66c457ad0
--- /dev/null
+++ b/src/tools/linkchecker/tests/broken_fragment_local/foo.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+<a href="#somefrag">test</a>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/broken_fragment_remote/bar.html b/src/tools/linkchecker/tests/broken_fragment_remote/bar.html
new file mode 100644
index 000000000..7879e1ce9
--- /dev/null
+++ b/src/tools/linkchecker/tests/broken_fragment_remote/bar.html
@@ -0,0 +1,4 @@
+<html>
+<body>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/broken_fragment_remote/inner/foo.html b/src/tools/linkchecker/tests/broken_fragment_remote/inner/foo.html
new file mode 100644
index 000000000..7683060b3
--- /dev/null
+++ b/src/tools/linkchecker/tests/broken_fragment_remote/inner/foo.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+<a href="../bar.html#somefrag">test</a>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/broken_redir/foo.html b/src/tools/linkchecker/tests/broken_redir/foo.html
new file mode 100644
index 000000000..bd3e3ad33
--- /dev/null
+++ b/src/tools/linkchecker/tests/broken_redir/foo.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+ <a href="redir-bad.html">bad redir</a>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/broken_redir/redir-bad.html b/src/tools/linkchecker/tests/broken_redir/redir-bad.html
new file mode 100644
index 000000000..9c580d8e0
--- /dev/null
+++ b/src/tools/linkchecker/tests/broken_redir/redir-bad.html
@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta http-equiv="refresh" content="0;URL=sometarget">
+ <title>Redirection</title>
+</head>
+<body>
+ <p>Redirecting to <a href="sometarget">sometarget</a>...</p>
+ <script>location.replace("sometarget" + location.search + location.hash);</script>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/checks.rs b/src/tools/linkchecker/tests/checks.rs
new file mode 100644
index 000000000..1a0b1b00e
--- /dev/null
+++ b/src/tools/linkchecker/tests/checks.rs
@@ -0,0 +1,113 @@
+use std::path::Path;
+use std::process::{Command, ExitStatus};
+
+fn run(dirname: &str) -> (ExitStatus, String, String) {
+ let output = Command::new(env!("CARGO_BIN_EXE_linkchecker"))
+ .current_dir(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests"))
+ .arg(dirname)
+ .output()
+ .unwrap();
+ let stdout = String::from_utf8(output.stdout).unwrap();
+ let stderr = String::from_utf8(output.stderr).unwrap();
+ (output.status, stdout, stderr)
+}
+
+fn broken_test(dirname: &str, expected: &str) {
+ let (status, stdout, stderr) = run(dirname);
+ assert!(!status.success());
+ if !contains(expected, &stdout) {
+ panic!(
+ "stdout did not contain expected text: {}\n\
+ --- stdout:\n\
+ {}\n\
+ --- stderr:\n\
+ {}\n",
+ expected, stdout, stderr
+ );
+ }
+}
+
+fn contains(expected: &str, actual: &str) -> bool {
+ // Normalize for Windows paths.
+ let actual = actual.replace('\\', "/");
+ actual.lines().any(|mut line| {
+ for (i, part) in expected.split("[..]").enumerate() {
+ match line.find(part) {
+ Some(j) => {
+ if i == 0 && j != 0 {
+ return false;
+ }
+ line = &line[j + part.len()..];
+ }
+ None => return false,
+ }
+ }
+ line.is_empty() || expected.ends_with("[..]")
+ })
+}
+
+fn valid_test(dirname: &str) {
+ let (status, stdout, stderr) = run(dirname);
+ if !status.success() {
+ panic!(
+ "test did not succeed as expected\n\
+ --- stdout:\n\
+ {}\n\
+ --- stderr:\n\
+ {}\n",
+ stdout, stderr
+ );
+ }
+}
+
+#[test]
+fn valid() {
+ valid_test("valid/inner");
+}
+
+#[test]
+fn basic_broken() {
+ broken_test("basic_broken", "foo.html:3: broken link - `bar.html`");
+}
+
+#[test]
+fn broken_fragment_local() {
+ broken_test(
+ "broken_fragment_local",
+ "foo.html:3: broken link fragment `#somefrag` pointing to `foo.html`",
+ );
+}
+
+#[test]
+fn broken_fragment_remote() {
+ broken_test(
+ "broken_fragment_remote/inner",
+ "foo.html:3: broken link fragment `#somefrag` pointing to \
+ `[..]/broken_fragment_remote/bar.html`",
+ );
+}
+
+#[test]
+fn broken_redir() {
+ broken_test(
+ "broken_redir",
+ "foo.html:3: broken redirect from `redir-bad.html` to `sometarget`",
+ );
+}
+
+#[test]
+fn directory_link() {
+ broken_test(
+ "directory_link",
+ "foo.html:3: directory link to `somedir` (directory links should use index.html instead)",
+ );
+}
+
+#[test]
+fn redirect_loop() {
+ broken_test(
+ "redirect_loop",
+ "foo.html:3: redirect from `redir-bad.html` to `[..]redirect_loop/redir-bad.html` \
+ which is also a redirect (not supported)",
+ );
+}
diff --git a/src/tools/linkchecker/tests/directory_link/foo.html b/src/tools/linkchecker/tests/directory_link/foo.html
new file mode 100644
index 000000000..40a8461b8
--- /dev/null
+++ b/src/tools/linkchecker/tests/directory_link/foo.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+ <a href="somedir">dir link</a>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/directory_link/somedir/index.html b/src/tools/linkchecker/tests/directory_link/somedir/index.html
new file mode 100644
index 000000000..7879e1ce9
--- /dev/null
+++ b/src/tools/linkchecker/tests/directory_link/somedir/index.html
@@ -0,0 +1,4 @@
+<html>
+<body>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/redirect_loop/foo.html b/src/tools/linkchecker/tests/redirect_loop/foo.html
new file mode 100644
index 000000000..bee58b212
--- /dev/null
+++ b/src/tools/linkchecker/tests/redirect_loop/foo.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+ <a href="redir-bad.html">loop link</a>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/redirect_loop/redir-bad.html b/src/tools/linkchecker/tests/redirect_loop/redir-bad.html
new file mode 100644
index 000000000..bc567caa7
--- /dev/null
+++ b/src/tools/linkchecker/tests/redirect_loop/redir-bad.html
@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta http-equiv="refresh" content="0;URL=redir-bad.html">
+ <title>Redirection</title>
+</head>
+<body>
+ <p>Redirecting to <a href="redir-bad.html">redir-bad.html</a>...</p>
+ <script>location.replace("redir-bad.html" + location.search + location.hash);</script>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/valid/inner/bar.html b/src/tools/linkchecker/tests/valid/inner/bar.html
new file mode 100644
index 000000000..4b500d78b
--- /dev/null
+++ b/src/tools/linkchecker/tests/valid/inner/bar.html
@@ -0,0 +1,7 @@
+<html>
+<body>
+
+ <h2 id="barfrag">Bar</h2>
+
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/valid/inner/foo.html b/src/tools/linkchecker/tests/valid/inner/foo.html
new file mode 100644
index 000000000..3c6a7483b
--- /dev/null
+++ b/src/tools/linkchecker/tests/valid/inner/foo.html
@@ -0,0 +1,14 @@
+<html>
+<body>
+ <a href="#localfrag">test local frag</a>
+ <a href="../outer.html">remote link</a>
+ <a href="../outer.html#somefrag">remote link with fragment</a>
+ <a href="bar.html">this book</a>
+ <a href="bar.html#barfrag">this book with fragment</a>
+ <a href="https://example.com/doesnotexist">external links not validated</a>
+ <a href="redir.html#redirfrag">Redirect</a>
+
+ <h2 id="localfrag">Local</h2>
+
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/valid/inner/redir-bad.html b/src/tools/linkchecker/tests/valid/inner/redir-bad.html
new file mode 100644
index 000000000..f32683efe
--- /dev/null
+++ b/src/tools/linkchecker/tests/valid/inner/redir-bad.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta http-equiv="refresh" content="0;URL=xxx">
+ <title>Redirection</title>
+</head>
+<body>
+ <p>Redirecting to <a href="xxx">xxx</a>...</p>
+ <script>location.replace("xxx" + location.search + location.hash);</script>
+ These files are skipped, but probably shouldn't be.
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/valid/inner/redir-target.html b/src/tools/linkchecker/tests/valid/inner/redir-target.html
new file mode 100644
index 000000000..bd59884a0
--- /dev/null
+++ b/src/tools/linkchecker/tests/valid/inner/redir-target.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+ <h2 id="redirfrag">Redir</h2>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/valid/inner/redir.html b/src/tools/linkchecker/tests/valid/inner/redir.html
new file mode 100644
index 000000000..3a52a8973
--- /dev/null
+++ b/src/tools/linkchecker/tests/valid/inner/redir.html
@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta http-equiv="refresh" content="0;URL=redir-target.html">
+ <title>Redirection</title>
+</head>
+<body>
+ <p>Redirecting to <a href="redir-target.html">redir-target.html</a>...</p>
+ <script>location.replace("redir-target.html" + location.search + location.hash);</script>
+</body>
+</html>
diff --git a/src/tools/linkchecker/tests/valid/outer.html b/src/tools/linkchecker/tests/valid/outer.html
new file mode 100644
index 000000000..35f799f20
--- /dev/null
+++ b/src/tools/linkchecker/tests/valid/outer.html
@@ -0,0 +1,5 @@
+<html>
+<body>
+<a id="somefrag"></a>
+</body>
+</html>