diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-17 12:02:58 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-17 12:02:58 +0000 |
commit | 698f8c2f01ea549d77d7dc3338a12e04c11057b9 (patch) | |
tree | 173a775858bd501c378080a10dca74132f05bc50 /src/doc/book/tools | |
parent | Initial commit. (diff) | |
download | rustc-698f8c2f01ea549d77d7dc3338a12e04c11057b9.tar.xz rustc-698f8c2f01ea549d77d7dc3338a12e04c11057b9.zip |
Adding upstream version 1.64.0+dfsg1.upstream/1.64.0+dfsg1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/doc/book/tools')
-rwxr-xr-x | src/doc/book/tools/convert-quotes.sh | 12 | ||||
-rwxr-xr-x | src/doc/book/tools/doc-to-md.sh | 20 | ||||
-rw-r--r-- | src/doc/book/tools/docx-to-md.xsl | 220 | ||||
-rwxr-xr-x | src/doc/book/tools/megadiff.sh | 22 | ||||
-rwxr-xr-x | src/doc/book/tools/nostarch.sh | 27 | ||||
-rw-r--r-- | src/doc/book/tools/src/bin/concat_chapters.rs | 127 | ||||
-rw-r--r-- | src/doc/book/tools/src/bin/convert_quotes.rs | 78 | ||||
-rw-r--r-- | src/doc/book/tools/src/bin/lfp.rs | 248 | ||||
-rw-r--r-- | src/doc/book/tools/src/bin/link2print.rs | 415 | ||||
-rw-r--r-- | src/doc/book/tools/src/bin/release_listings.rs | 159 | ||||
-rw-r--r-- | src/doc/book/tools/src/bin/remove_hidden_lines.rs | 83 | ||||
-rw-r--r-- | src/doc/book/tools/src/bin/remove_links.rs | 45 | ||||
-rw-r--r-- | src/doc/book/tools/src/bin/remove_markup.rs | 53 | ||||
-rwxr-xr-x | src/doc/book/tools/update-editions.sh | 8 | ||||
-rwxr-xr-x | src/doc/book/tools/update-rustc.sh | 93 |
15 files changed, 1610 insertions, 0 deletions
diff --git a/src/doc/book/tools/convert-quotes.sh b/src/doc/book/tools/convert-quotes.sh new file mode 100755 index 000000000..bffe82359 --- /dev/null +++ b/src/doc/book/tools/convert-quotes.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -eu + +mkdir -p tmp/src +rm -rf tmp/*.md + +for f in src/"${1:-\"\"}"*.md +do + cargo run --bin convert_quotes < "$f" > "tmp/$f" + mv "tmp/$f" "$f" +done diff --git a/src/doc/book/tools/doc-to-md.sh b/src/doc/book/tools/doc-to-md.sh new file mode 100755 index 000000000..cffe4c04b --- /dev/null +++ b/src/doc/book/tools/doc-to-md.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -eu + +# Get all the docx files in the tmp dir. +find tmp -name '*.docx' -print0 | \ +# Extract just the filename so we can reuse it easily. +xargs -0 basename -s .docx | \ +while IFS= read -r filename; do + # Make a directory to put the XML in. + mkdir -p "tmp/$filename" + # Unzip the docx to get at the XML. + unzip -o "tmp/$filename.docx" -d "tmp/$filename" + # Convert to markdown with XSL. + xsltproc tools/docx-to-md.xsl "tmp/$filename/word/document.xml" | \ + # Hard wrap at 80 chars at word bourdaries. + fold -w 80 -s | \ + # Remove trailing whitespace and save in the `nostarch` dir for comparison. + sed -e "s/ *$//" > "nostarch/$filename.md" +done diff --git a/src/doc/book/tools/docx-to-md.xsl b/src/doc/book/tools/docx-to-md.xsl new file mode 100644 index 000000000..637c7a59c --- /dev/null +++ b/src/doc/book/tools/docx-to-md.xsl @@ -0,0 +1,220 @@ +<?xml version="1.0"?> +<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:o="urn:schemas-microsoft-com:office:office" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:v="urn:schemas-microsoft-com:vml" xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main" xmlns:w10="urn:schemas-microsoft-com:office:word" xmlns:wp="http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing" xmlns:wps="http://schemas.microsoft.com/office/word/2010/wordprocessingShape" xmlns:wpg="http://schemas.microsoft.com/office/word/2010/wordprocessingGroup" xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" xmlns:wp14="http://schemas.microsoft.com/office/word/2010/wordprocessingDrawing" xmlns:w14="http://schemas.microsoft.com/office/word/2010/wordml"> + <xsl:output method="text" /> + <xsl:template match="/"> + <xsl:apply-templates select="/w:document/w:body/*" /> + </xsl:template> + + <!-- Ignore these --> + <xsl:template match="w:p[starts-with(w:pPr/w:pStyle/@w:val, 'TOC')]" /> + <xsl:template match="w:p[starts-with(w:pPr/w:pStyle/@w:val, 'Contents1')]" /> + <xsl:template match="w:p[starts-with(w:pPr/w:pStyle/@w:val, 'Contents2')]" /> + <xsl:template match="w:p[starts-with(w:pPr/w:pStyle/@w:val, 'Contents3')]" /> + + <xsl:template match="w:p[w:pPr/w:pStyle/@w:val = 'ChapterStart']" /> + <xsl:template match="w:p[w:pPr/w:pStyle/@w:val = 'Normal']" /> + <xsl:template match="w:p[w:pPr/w:pStyle/@w:val = 'Standard']" /> + <xsl:template match="w:p[w:pPr/w:pStyle/@w:val = 'AuthorQuery']" /> + + <xsl:template match="w:p[w:pPr[not(w:pStyle)]]" /> + + <!-- Paragraph styles --> + + <xsl:template match="w:p[w:pPr/w:pStyle/@w:val = 'ChapterTitle']"> + <xsl:text> [TOC] </xsl:text> + <xsl:text># </xsl:text> + <xsl:apply-templates select="*" /> + <xsl:text> </xsl:text> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle/@w:val = 'HeadA']"> + <xsl:text>## </xsl:text> + <xsl:apply-templates select="*" /> + <xsl:text> </xsl:text> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle/@w:val = 'HeadB']"> + <xsl:text>### </xsl:text> + <xsl:apply-templates select="*" /> + <xsl:text> </xsl:text> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle/@w:val = 'HeadC']"> + <xsl:text>#### </xsl:text> + <xsl:apply-templates select="*" /> + <xsl:text> </xsl:text> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle/@w:val = 'HeadBox']"> + <xsl:text>### </xsl:text> + <xsl:apply-templates select="*" /> + <xsl:text> </xsl:text> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle[@w:val = 'NumListA' or @w:val = 'NumListB']]"> + <xsl:text>1. </xsl:text> + <xsl:apply-templates select="*" /> + <xsl:text> </xsl:text> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle[@w:val = 'NumListC']]"> + <xsl:text>1. </xsl:text> + <xsl:apply-templates select="*" /> + <xsl:text> </xsl:text> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle[@w:val = 'BulletA' or @w:val = 'BulletB' or @w:val = 'ListPlainA' or @w:val = 'ListPlainB']]"> + <xsl:text>* </xsl:text> + <xsl:apply-templates select="*" /> + <xsl:text> </xsl:text> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle[@w:val = 'BulletC' or @w:val = 'ListPlainC']]"> + <xsl:text>* </xsl:text> + <xsl:apply-templates select="*" /> + <xsl:text> </xsl:text> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle[@w:val = 'SubBullet']]"> + <xsl:text> * </xsl:text> + <xsl:apply-templates select="*" /> + <xsl:text> </xsl:text> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle[@w:val = 'BodyFirst' or @w:val = 'Body' or @w:val = 'BodyFirstBox' or @w:val = 'BodyBox' or @w:val = '1stPara']]"> + <xsl:if test=".//w:t"> + <xsl:apply-templates select="*" /> + <xsl:text> </xsl:text> + </xsl:if> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle[@w:val = 'CodeA' or @w:val = 'CodeAWingding']]"> + <xsl:text>``` </xsl:text> + <!-- Don't apply Emphasis/etc templates in code blocks --> + <xsl:for-each select="w:r"> + <xsl:value-of select="w:t" /> + </xsl:for-each> + <xsl:text> </xsl:text> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle[@w:val = 'CodeB' or @w:val = 'CodeBWingding']]"> + <!-- Don't apply Emphasis/etc templates in code blocks --> + <xsl:for-each select="w:r"> + <xsl:value-of select="w:t" /> + </xsl:for-each> + <xsl:text> </xsl:text> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle[@w:val = 'CodeC' or @w:val = 'CodeCWingding']]"> + <!-- Don't apply Emphasis/etc templates in code blocks --> + <xsl:for-each select="w:r"> + <xsl:value-of select="w:t" /> + </xsl:for-each> + <xsl:text> ``` </xsl:text> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle/@w:val = 'CodeSingle']"> + <xsl:text>``` </xsl:text> + <xsl:apply-templates select="*" /> + <xsl:text> ``` </xsl:text> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle/@w:val = 'ProductionDirective']"> + <xsl:apply-templates select="*" /> + <xsl:text> </xsl:text> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle[@w:val = 'Caption' or @w:val = 'TableTitle' or @w:val = 'Caption1' or @w:val = 'Listing']]"> + <xsl:apply-templates select="*" /> + <xsl:text> </xsl:text> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle[@w:val = 'BlockQuote']]"> + <xsl:text>> </xsl:text> + <xsl:apply-templates select="*" /> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle[@w:val = 'BlockText']]"> + <xsl:text> </xsl:text> + <xsl:text>> </xsl:text> + <xsl:apply-templates select="*" /> + <xsl:text> </xsl:text> + </xsl:template> + + <xsl:template match="w:p[w:pPr/w:pStyle/@w:val = 'Note']"> + <xsl:text>> </xsl:text> + <xsl:apply-templates select="*" /> + <xsl:text> </xsl:text> + </xsl:template> + + <xsl:template match="w:p"> +Unmatched: <xsl:value-of select="w:pPr/w:pStyle/@w:val" /> + <xsl:text> + </xsl:text> + + + </xsl:template> + + <!-- Character styles --> + + <xsl:template match="w:r[w:rPr/w:rStyle[@w:val = 'Literal' or @w:val = 'LiteralBold' or @w:val = 'LiteralCaption' or @w:val = 'LiteralBox']]"> + <xsl:choose> + <xsl:when test="normalize-space(w:t) != ''"> + <xsl:if test="starts-with(w:t, ' ')"> + <xsl:text> </xsl:text> + </xsl:if> + <xsl:text>`</xsl:text> + <xsl:value-of select="normalize-space(w:t)" /> + <xsl:text>`</xsl:text> + <xsl:if test="substring(w:t, string-length(w:t)) = ' '"> + <xsl:text> </xsl:text> + </xsl:if> + </xsl:when> + <xsl:when test="normalize-space(w:t) != w:t and w:t != ''"> + <xsl:text> </xsl:text> + </xsl:when> + </xsl:choose> + </xsl:template> + + <xsl:template match="w:r[w:rPr/w:rStyle[@w:val = 'EmphasisBold']]"> + <xsl:choose> + <xsl:when test="normalize-space(w:t) != ''"> + <xsl:if test="starts-with(w:t, ' ')"> + <xsl:text> </xsl:text> + </xsl:if> + <xsl:text>**</xsl:text> + <xsl:value-of select="normalize-space(w:t)" /> + <xsl:text>**</xsl:text> + <xsl:if test="substring(w:t, string-length(w:t)) = ' '"> + <xsl:text> </xsl:text> + </xsl:if> + </xsl:when> + <xsl:when test="normalize-space(w:t) != w:t and w:t != ''"> + <xsl:text> </xsl:text> + </xsl:when> + </xsl:choose> + </xsl:template> + + <xsl:template match="w:r[w:rPr/w:rStyle[@w:val = 'EmphasisItalic' or @w:val = 'EmphasisItalicBox' or @w:val = 'EmphasisNote' or @w:val = 'EmphasisRevCaption' or @w:val = 'EmphasisRevItal']]"> + <xsl:choose> + <xsl:when test="normalize-space(w:t) != ''"> + <xsl:if test="starts-with(w:t, ' ')"> + <xsl:text> </xsl:text> + </xsl:if> + <xsl:text>*</xsl:text> + <xsl:value-of select="normalize-space(w:t)" /> + <xsl:text>*</xsl:text> + <xsl:if test="substring(w:t, string-length(w:t)) = ' '"> + <xsl:text> </xsl:text> + </xsl:if> + </xsl:when> + <xsl:otherwise> + <xsl:text> </xsl:text> + </xsl:otherwise> + </xsl:choose> + </xsl:template> + + <xsl:template match="w:r"> + <xsl:value-of select="w:t" /> + </xsl:template> +</xsl:stylesheet> diff --git a/src/doc/book/tools/megadiff.sh b/src/doc/book/tools/megadiff.sh new file mode 100755 index 000000000..f1e510249 --- /dev/null +++ b/src/doc/book/tools/megadiff.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -eu + +# Remove files that are never affected by rustfmt or are otherwise uninteresting +rm -rf tmp/book-before/css/ tmp/book-before/theme/ tmp/book-before/img/ tmp/book-before/*.js \ + tmp/book-before/FontAwesome tmp/book-before/*.css tmp/book-before/*.png \ + tmp/book-before/*.json tmp/book-before/print.html + +rm -rf tmp/book-after/css/ tmp/book-after/theme/ tmp/book-after/img/ tmp/book-after/*.js \ + tmp/book-after/FontAwesome tmp/book-after/*.css tmp/book-after/*.png \ + tmp/book-after/*.json tmp/book-after/print.html + +# Get all the html files before +find tmp/book-before -name '*.html' -print0 | \ +# Extract just the filename so we can reuse it easily. +xargs -0 basename | \ +while IFS= read -r filename; do + # Remove any files that are the same before and after + diff "tmp/book-before/$filename" "tmp/book-after/$filename" > /dev/null \ + && rm "tmp/book-before/$filename" "tmp/book-after/$filename" +done diff --git a/src/doc/book/tools/nostarch.sh b/src/doc/book/tools/nostarch.sh new file mode 100755 index 000000000..eec0ac5ea --- /dev/null +++ b/src/doc/book/tools/nostarch.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -eu + +cargo build --release + +mkdir -p tmp +rm -rf tmp/*.md +rm -rf tmp/markdown + +# Render the book as Markdown to include all the code listings +MDBOOK_OUTPUT__MARKDOWN=1 mdbook build -d tmp + +# Get all the Markdown files +find tmp/markdown -name "${1:-\"\"}*.md" -print0 | \ +# Extract just the filename so we can reuse it easily. +xargs -0 basename | \ +# Remove all links followed by `<!-- ignore -->``, then +# Change all remaining links from Markdown to italicized inline text. +while IFS= read -r filename; do + < "tmp/markdown/$filename" ./target/release/remove_links \ + | ./target/release/link2print \ + | ./target/release/remove_markup \ + | ./target/release/remove_hidden_lines > "tmp/$filename" +done +# Concatenate the files into the `nostarch` dir. +./target/release/concat_chapters tmp nostarch diff --git a/src/doc/book/tools/src/bin/concat_chapters.rs b/src/doc/book/tools/src/bin/concat_chapters.rs new file mode 100644 index 000000000..79ffec9b7 --- /dev/null +++ b/src/doc/book/tools/src/bin/concat_chapters.rs @@ -0,0 +1,127 @@ +#[macro_use] +extern crate lazy_static; + +use std::collections::BTreeMap; +use std::env; +use std::fs::{create_dir, read_dir, File}; +use std::io; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::exit; + +use regex::Regex; + +static PATTERNS: &[(&str, &str)] = &[ + (r"ch(\d\d)-\d\d-.*\.md", "chapter$1.md"), + (r"appendix-(\d\d).*\.md", "appendix.md"), +]; + +lazy_static! { + static ref MATCHERS: Vec<(Regex, &'static str)> = { + PATTERNS + .iter() + .map(|&(expr, repl)| (Regex::new(expr).unwrap(), repl)) + .collect() + }; +} + +fn main() { + let args: Vec<String> = env::args().collect(); + + if args.len() < 3 { + println!("Usage: {} <src-dir> <target-dir>", args[0]); + exit(1); + } + + let source_dir = ensure_dir_exists(&args[1]).unwrap(); + let target_dir = ensure_dir_exists(&args[2]).unwrap(); + + let mut matched_files = match_files(source_dir, target_dir); + matched_files.sort(); + + for (target_path, source_paths) in group_by_target(matched_files) { + concat_files(source_paths, target_path).unwrap(); + } +} + +fn match_files( + source_dir: &Path, + target_dir: &Path, +) -> Vec<(PathBuf, PathBuf)> { + read_dir(source_dir) + .expect("Unable to read source directory") + .filter_map(|maybe_entry| maybe_entry.ok()) + .filter_map(|entry| { + let source_filename = entry.file_name(); + let source_filename = + &source_filename.to_string_lossy().into_owned(); + for &(ref regex, replacement) in MATCHERS.iter() { + if regex.is_match(source_filename) { + let target_filename = + regex.replace_all(source_filename, replacement); + let source_path = entry.path(); + let mut target_path = PathBuf::from(&target_dir); + target_path.push(target_filename.to_string()); + return Some((source_path, target_path)); + } + } + None + }) + .collect() +} + +fn group_by_target( + matched_files: Vec<(PathBuf, PathBuf)>, +) -> BTreeMap<PathBuf, Vec<PathBuf>> { + let mut grouped: BTreeMap<PathBuf, Vec<PathBuf>> = BTreeMap::new(); + for (source, target) in matched_files { + if let Some(source_paths) = grouped.get_mut(&target) { + source_paths.push(source); + continue; + } + let source_paths = vec![source]; + grouped.insert(target.clone(), source_paths); + } + grouped +} + +fn concat_files( + source_paths: Vec<PathBuf>, + target_path: PathBuf, +) -> io::Result<()> { + println!("Concatenating into {}:", target_path.to_string_lossy()); + let mut target = File::create(target_path)?; + + write!( + target, + "\ +<!-- DO NOT EDIT THIS FILE. + +This file is periodically generated from the content in the `/src/` +directory, so all fixes need to be made in `/src/`. +--> + +[TOC] +" + )?; + + for path in source_paths { + println!(" {}", path.to_string_lossy()); + let mut source = File::open(path)?; + let mut contents: Vec<u8> = Vec::new(); + source.read_to_end(&mut contents)?; + + target.write_all(b"\n")?; + target.write_all(&contents)?; + target.write_all(b"\n")?; + } + Ok(()) +} + +fn ensure_dir_exists(dir_string: &str) -> io::Result<&Path> { + let path = Path::new(dir_string); + if !path.exists() { + create_dir(path)?; + } + Ok(path) +} diff --git a/src/doc/book/tools/src/bin/convert_quotes.rs b/src/doc/book/tools/src/bin/convert_quotes.rs new file mode 100644 index 000000000..b4a9bdce2 --- /dev/null +++ b/src/doc/book/tools/src/bin/convert_quotes.rs @@ -0,0 +1,78 @@ +use std::io; +use std::io::Read; + +fn main() { + let mut is_in_code_block = false; + let mut is_in_inline_code = false; + let mut is_in_html_tag = false; + + let mut buffer = String::new(); + if let Err(e) = io::stdin().read_to_string(&mut buffer) { + panic!("{}", e); + } + + for line in buffer.lines() { + if line.is_empty() { + is_in_inline_code = false; + } + if line.starts_with("```") { + is_in_code_block = !is_in_code_block; + } + if is_in_code_block { + is_in_inline_code = false; + is_in_html_tag = false; + println!("{}", line); + } else { + let modified_line = &mut String::new(); + let mut previous_char = std::char::REPLACEMENT_CHARACTER; + let chars_in_line = line.chars(); + + for possible_match in chars_in_line { + // Check if inside inline code. + if possible_match == '`' { + is_in_inline_code = !is_in_inline_code; + } + // Check if inside HTML tag. + if possible_match == '<' && !is_in_inline_code { + is_in_html_tag = true; + } + if possible_match == '>' && !is_in_inline_code { + is_in_html_tag = false; + } + + // Replace with right/left apostrophe/quote. + let char_to_push = if possible_match == '\'' + && !is_in_inline_code + && !is_in_html_tag + { + if (previous_char != std::char::REPLACEMENT_CHARACTER + && !previous_char.is_whitespace()) + || previous_char == '‘' + { + '’' + } else { + '‘' + } + } else if possible_match == '"' + && !is_in_inline_code + && !is_in_html_tag + { + if (previous_char != std::char::REPLACEMENT_CHARACTER + && !previous_char.is_whitespace()) + || previous_char == '“' + { + '”' + } else { + '“' + } + } else { + // Leave untouched. + possible_match + }; + modified_line.push(char_to_push); + previous_char = char_to_push; + } + println!("{}", modified_line); + } + } +} diff --git a/src/doc/book/tools/src/bin/lfp.rs b/src/doc/book/tools/src/bin/lfp.rs new file mode 100644 index 000000000..c4d4bce03 --- /dev/null +++ b/src/doc/book/tools/src/bin/lfp.rs @@ -0,0 +1,248 @@ +// We have some long regex literals, so: +// ignore-tidy-linelength + +use docopt::Docopt; +use serde::Deserialize; +use std::io::BufRead; +use std::{fs, io, path}; + +fn main() { + let args: Args = Docopt::new(USAGE) + .and_then(|d| d.deserialize()) + .unwrap_or_else(|e| e.exit()); + + let src_dir = &path::Path::new(&args.arg_src_dir); + let found_errs = walkdir::WalkDir::new(src_dir) + .min_depth(1) + .into_iter() + .map(|entry| match entry { + Ok(entry) => entry, + Err(err) => { + eprintln!("{:?}", err); + std::process::exit(911) + } + }) + .map(|entry| { + let path = entry.path(); + if is_file_of_interest(path) { + let err_vec = lint_file(path); + for err in &err_vec { + match *err { + LintingError::LineOfInterest(line_num, ref line) => { + eprintln!( + "{}:{}\t{}", + path.display(), + line_num, + line + ) + } + LintingError::UnableToOpenFile => { + eprintln!("Unable to open {}.", path.display()) + } + } + } + !err_vec.is_empty() + } else { + false + } + }) + .collect::<Vec<_>>() + .iter() + .any(|result| *result); + + if found_errs { + std::process::exit(1) + } else { + std::process::exit(0) + } +} + +const USAGE: &str = " +counter +Usage: + lfp <src-dir> + lfp (-h | --help) +Options: + -h --help Show this screen. +"; + +#[derive(Debug, Deserialize)] +struct Args { + arg_src_dir: String, +} + +fn lint_file(path: &path::Path) -> Vec<LintingError> { + match fs::File::open(path) { + Ok(file) => lint_lines(io::BufReader::new(&file).lines()), + Err(_) => vec![LintingError::UnableToOpenFile], + } +} + +fn lint_lines<I>(lines: I) -> Vec<LintingError> +where + I: Iterator<Item = io::Result<String>>, +{ + lines + .enumerate() + .map(|(line_num, line)| { + let raw_line = line.unwrap(); + if is_line_of_interest(&raw_line) { + Err(LintingError::LineOfInterest(line_num, raw_line)) + } else { + Ok(()) + } + }) + .filter(|result| result.is_err()) + .map(|result| result.unwrap_err()) + .collect() +} + +fn is_file_of_interest(path: &path::Path) -> bool { + path.extension().map_or(false, |ext| ext == "md") +} + +fn is_line_of_interest(line: &str) -> bool { + line.split_whitespace().any(|sub_string| { + sub_string.contains("file://") + && !sub_string.contains("file:///projects/") + }) +} + +#[derive(Debug)] +enum LintingError { + UnableToOpenFile, + LineOfInterest(usize, String), +} + +#[cfg(test)] +mod tests { + + use std::path; + + #[test] + fn lint_file_returns_a_vec_with_errs_when_lines_of_interest_are_found() { + let string = r#" + $ cargo run + Compiling guessing_game v0.1.0 (file:///home/you/projects/guessing_game) + Running `target/guessing_game` + Guess the number! + The secret number is: 61 + Please input your guess. + 10 + You guessed: 10 + Too small! + Please input your guess. + 99 + You guessed: 99 + Too big! + Please input your guess. + foo + Please input your guess. + 61 + You guessed: 61 + You win! + $ cargo run + Compiling guessing_game v0.1.0 (file:///home/you/projects/guessing_game) + Running `target/debug/guessing_game` + Guess the number! + The secret number is: 7 + Please input your guess. + 4 + You guessed: 4 + $ cargo run + Running `target/debug/guessing_game` + Guess the number! + The secret number is: 83 + Please input your guess. + 5 + $ cargo run + Compiling guessing_game v0.1.0 (file:///home/you/projects/guessing_game) + Running `target/debug/guessing_game` + Hello, world! + "#; + + let raw_lines = string.to_string(); + let lines = raw_lines.lines().map(|line| Ok(line.to_string())); + + let result_vec = super::lint_lines(lines); + + assert!(!result_vec.is_empty()); + assert_eq!(3, result_vec.len()); + } + + #[test] + fn lint_file_returns_an_empty_vec_when_no_lines_of_interest_are_found() { + let string = r#" + $ cargo run + Compiling guessing_game v0.1.0 (file:///projects/guessing_game) + Running `target/guessing_game` + Guess the number! + The secret number is: 61 + Please input your guess. + 10 + You guessed: 10 + Too small! + Please input your guess. + 99 + You guessed: 99 + Too big! + Please input your guess. + foo + Please input your guess. + 61 + You guessed: 61 + You win! + "#; + + let raw_lines = string.to_string(); + let lines = raw_lines.lines().map(|line| Ok(line.to_string())); + + let result_vec = super::lint_lines(lines); + + assert!(result_vec.is_empty()); + } + + #[test] + fn is_file_of_interest_returns_false_when_the_path_is_a_directory() { + let uninteresting_fn = "src/img"; + + assert!(!super::is_file_of_interest(path::Path::new( + uninteresting_fn + ))); + } + + #[test] + fn is_file_of_interest_returns_false_when_the_filename_does_not_have_the_md_extension( + ) { + let uninteresting_fn = "src/img/foo1.png"; + + assert!(!super::is_file_of_interest(path::Path::new( + uninteresting_fn + ))); + } + + #[test] + fn is_file_of_interest_returns_true_when_the_filename_has_the_md_extension() + { + let interesting_fn = "src/ch01-00-introduction.md"; + + assert!(super::is_file_of_interest(path::Path::new(interesting_fn))); + } + + #[test] + fn is_line_of_interest_does_not_report_a_line_if_the_line_contains_a_file_url_which_is_directly_followed_by_the_project_path( + ) { + let sample_line = + "Compiling guessing_game v0.1.0 (file:///projects/guessing_game)"; + + assert!(!super::is_line_of_interest(sample_line)); + } + + #[test] + fn is_line_of_interest_reports_a_line_if_the_line_contains_a_file_url_which_is_not_directly_followed_by_the_project_path( + ) { + let sample_line = "Compiling guessing_game v0.1.0 (file:///home/you/projects/guessing_game)"; + + assert!(super::is_line_of_interest(sample_line)); + } +} diff --git a/src/doc/book/tools/src/bin/link2print.rs b/src/doc/book/tools/src/bin/link2print.rs new file mode 100644 index 000000000..1e92ecbcc --- /dev/null +++ b/src/doc/book/tools/src/bin/link2print.rs @@ -0,0 +1,415 @@ +// FIXME: we have some long lines that could be refactored, but it's not a big deal. +// ignore-tidy-linelength + +use regex::{Captures, Regex}; +use std::collections::HashMap; +use std::io; +use std::io::Read; + +fn main() { + write_md(parse_links(parse_references(read_md()))); +} + +fn read_md() -> String { + let mut buffer = String::new(); + match io::stdin().read_to_string(&mut buffer) { + Ok(_) => buffer, + Err(error) => panic!("{}", error), + } +} + +fn write_md(output: String) { + print!("{}", output); +} + +fn parse_references(buffer: String) -> (String, HashMap<String, String>) { + let mut ref_map = HashMap::new(); + // FIXME: currently doesn't handle "title" in following line. + let re = Regex::new(r###"(?m)\n?^ {0,3}\[([^]]+)\]:[[:blank:]]*(.*)$"###) + .unwrap(); + let output = re + .replace_all(&buffer, |caps: &Captures<'_>| { + let key = caps.get(1).unwrap().as_str().to_uppercase(); + let val = caps.get(2).unwrap().as_str().to_string(); + if ref_map.insert(key, val).is_some() { + panic!( + "Did not expect markdown page to have duplicate reference" + ); + } + "".to_string() + }) + .to_string(); + (output, ref_map) +} + +fn parse_links((buffer, ref_map): (String, HashMap<String, String>)) -> String { + // FIXME: check which punctuation is allowed by spec. + let re = Regex::new(r###"(?:(?P<pre>(?:```(?:[^`]|`[^`])*`?\n```\n)|(?:[^\[]`[^`\n]+[\n]?[^`\n]*`))|(?:\[(?P<name>[^]]+)\](?:(?:\([[:blank:]]*(?P<val>[^")]*[^ ])(?:[[:blank:]]*"[^"]*")?\))|(?:\[(?P<key>[^]]*)\]))?))"###).expect("could not create regex"); + let error_code = + Regex::new(r###"^E\d{4}$"###).expect("could not create regex"); + let output = re.replace_all(&buffer, |caps: &Captures<'_>| { + match caps.name("pre") { + Some(pre_section) => pre_section.as_str().to_string(), + None => { + let name = caps.name("name").expect("could not get name").as_str(); + // Really we should ignore text inside code blocks, + // this is a hack to not try to treat `#[derive()]`, + // `[profile]`, `[test]`, or `[E\d\d\d\d]` like a link. + if name.starts_with("derive(") || + name.starts_with("profile") || + name.starts_with("test") || + name.starts_with("no_mangle") || + error_code.is_match(name) { + return name.to_string() + } + + let val = match caps.name("val") { + // `[name](link)` + Some(value) => value.as_str().to_string(), + None => { + match caps.name("key") { + Some(key) => { + match key.as_str() { + // `[name][]` + "" => ref_map.get(&name.to_uppercase()).unwrap_or_else(|| panic!("could not find url for the link text `{}`", name)).to_string(), + // `[name][reference]` + _ => ref_map.get(&key.as_str().to_uppercase()).unwrap_or_else(|| panic!("could not find url for the link text `{}`", key.as_str())).to_string(), + } + } + // `[name]` as reference + None => ref_map.get(&name.to_uppercase()).unwrap_or_else(|| panic!("could not find url for the link text `{}`", name)).to_string(), + } + } + }; + format!("{} at *{}*", name, val) + } + } + }); + output.to_string() +} + +#[cfg(test)] +mod tests { + fn parse(source: String) -> String { + super::parse_links(super::parse_references(source)) + } + + #[test] + fn parses_inline_link() { + let source = + r"This is a [link](http://google.com) that should be expanded" + .to_string(); + let target = + r"This is a link at *http://google.com* that should be expanded" + .to_string(); + assert_eq!(parse(source), target); + } + + #[test] + fn parses_multiline_links() { + let source = r"This is a [link](http://google.com) that +should appear expanded. Another [location](/here/) and [another](http://gogogo)" + .to_string(); + let target = r"This is a link at *http://google.com* that +should appear expanded. Another location at */here/* and another at *http://gogogo*" + .to_string(); + assert_eq!(parse(source), target); + } + + #[test] + fn parses_reference() { + let source = r"This is a [link][theref]. +[theref]: http://example.com/foo +more text" + .to_string(); + let target = r"This is a link at *http://example.com/foo*. +more text" + .to_string(); + assert_eq!(parse(source), target); + } + + #[test] + fn parses_implicit_link() { + let source = r"This is an [implicit][] link. +[implicit]: /The Link/" + .to_string(); + let target = r"This is an implicit at */The Link/* link.".to_string(); + assert_eq!(parse(source), target); + } + #[test] + fn parses_refs_with_one_space_indentation() { + let source = r"This is a [link][ref] + [ref]: The link" + .to_string(); + let target = r"This is a link at *The link*".to_string(); + assert_eq!(parse(source), target); + } + + #[test] + fn parses_refs_with_two_space_indentation() { + let source = r"This is a [link][ref] + [ref]: The link" + .to_string(); + let target = r"This is a link at *The link*".to_string(); + assert_eq!(parse(source), target); + } + + #[test] + fn parses_refs_with_three_space_indentation() { + let source = r"This is a [link][ref] + [ref]: The link" + .to_string(); + let target = r"This is a link at *The link*".to_string(); + assert_eq!(parse(source), target); + } + + #[test] + #[should_panic] + fn rejects_refs_with_four_space_indentation() { + let source = r"This is a [link][ref] + [ref]: The link" + .to_string(); + let target = r"This is a link at *The link*".to_string(); + assert_eq!(parse(source), target); + } + + #[test] + fn ignores_optional_inline_title() { + let source = + r###"This is a titled [link](http://example.com "My title")."### + .to_string(); + let target = + r"This is a titled link at *http://example.com*.".to_string(); + assert_eq!(parse(source), target); + } + + #[test] + fn parses_title_with_puctuation() { + let source = + r###"[link](http://example.com "It's Title")"###.to_string(); + let target = r"link at *http://example.com*".to_string(); + assert_eq!(parse(source), target); + } + + #[test] + fn parses_name_with_punctuation() { + let source = r###"[I'm here](there)"###.to_string(); + let target = r###"I'm here at *there*"###.to_string(); + assert_eq!(parse(source), target); + } + #[test] + fn parses_name_with_utf8() { + let source = r###"[user’s forum](the user’s forum)"###.to_string(); + let target = r###"user’s forum at *the user’s forum*"###.to_string(); + assert_eq!(parse(source), target); + } + + #[test] + fn parses_reference_with_punctuation() { + let source = r###"[link][the ref-ref] +[the ref-ref]:http://example.com/ref-ref"### + .to_string(); + let target = r###"link at *http://example.com/ref-ref*"###.to_string(); + assert_eq!(parse(source), target); + } + + #[test] + fn parses_reference_case_insensitively() { + let source = r"[link][Ref] +[ref]: The reference" + .to_string(); + let target = r"link at *The reference*".to_string(); + assert_eq!(parse(source), target); + } + #[test] + fn parses_link_as_reference_when_reference_is_empty() { + let source = r"[link as reference][] +[link as reference]: the actual reference" + .to_string(); + let target = r"link as reference at *the actual reference*".to_string(); + assert_eq!(parse(source), target); + } + + #[test] + fn parses_link_without_reference_as_reference() { + let source = r"[link] is alone +[link]: The contents" + .to_string(); + let target = r"link at *The contents* is alone".to_string(); + assert_eq!(parse(source), target); + } + + #[test] + #[ignore] + fn parses_link_without_reference_as_reference_with_asterisks() { + let source = r"*[link]* is alone +[link]: The contents" + .to_string(); + let target = r"*link* at *The contents* is alone".to_string(); + assert_eq!(parse(source), target); + } + #[test] + fn ignores_links_in_pre_sections() { + let source = r###"```toml +[package] +name = "hello_cargo" +version = "0.1.0" + +[dependencies] +``` +"### + .to_string(); + let target = source.clone(); + assert_eq!(parse(source), target); + } + + #[test] + fn ignores_links_in_quoted_sections() { + let source = r###"do not change `[package]`."###.to_string(); + let target = source.clone(); + assert_eq!(parse(source), target); + } + #[test] + fn ignores_links_in_quoted_sections_containing_newlines() { + let source = r"do not change `this [package] +is still here` [link](ref)" + .to_string(); + let target = r"do not change `this [package] +is still here` link at *ref*" + .to_string(); + assert_eq!(parse(source), target); + } + + #[test] + fn ignores_links_in_pre_sections_while_still_handling_links() { + let source = r###"```toml +[package] +name = "hello_cargo" +version = "0.1.0" + +[dependencies] +``` +Another [link] +more text +[link]: http://gohere +"### + .to_string(); + let target = r###"```toml +[package] +name = "hello_cargo" +version = "0.1.0" + +[dependencies] +``` +Another link at *http://gohere* +more text +"### + .to_string(); + assert_eq!(parse(source), target); + } + #[test] + fn ignores_quotes_in_pre_sections() { + let source = r###"```bash +$ cargo build + Compiling guessing_game v0.1.0 (file:///projects/guessing_game) +src/main.rs:23:21: 23:35 error: mismatched types [E0308] +src/main.rs:23 match guess.cmp(&secret_number) { + ^~~~~~~~~~~~~~ +src/main.rs:23:21: 23:35 help: run `rustc --explain E0308` to see a detailed explanation +src/main.rs:23:21: 23:35 note: expected type `&std::string::String` +src/main.rs:23:21: 23:35 note: found type `&_` +error: aborting due to previous error +Could not compile `guessing_game`. +``` +"### + .to_string(); + let target = source.clone(); + assert_eq!(parse(source), target); + } + #[test] + fn ignores_short_quotes() { + let source = r"to `1` at index `[0]` i".to_string(); + let target = source.clone(); + assert_eq!(parse(source), target); + } + #[test] + fn ignores_pre_sections_with_final_quote() { + let source = r###"```bash +$ cargo run + Compiling points v0.1.0 (file:///projects/points) +error: the trait bound `Point: std::fmt::Display` is not satisfied [--explain E0277] + --> src/main.rs:8:29 +8 |> println!("Point 1: {}", p1); + |> ^^ +<std macros>:2:27: 2:58: note: in this expansion of format_args! +<std macros>:3:1: 3:54: note: in this expansion of print! (defined in <std macros>) +src/main.rs:8:5: 8:33: note: in this expansion of println! (defined in <std macros>) +note: `Point` cannot be formatted with the default formatter; try using `:?` instead if you are using a format string +note: required by `std::fmt::Display::fmt` +``` +`here` is another [link](the ref) +"###.to_string(); + let target = r###"```bash +$ cargo run + Compiling points v0.1.0 (file:///projects/points) +error: the trait bound `Point: std::fmt::Display` is not satisfied [--explain E0277] + --> src/main.rs:8:29 +8 |> println!("Point 1: {}", p1); + |> ^^ +<std macros>:2:27: 2:58: note: in this expansion of format_args! +<std macros>:3:1: 3:54: note: in this expansion of print! (defined in <std macros>) +src/main.rs:8:5: 8:33: note: in this expansion of println! (defined in <std macros>) +note: `Point` cannot be formatted with the default formatter; try using `:?` instead if you are using a format string +note: required by `std::fmt::Display::fmt` +``` +`here` is another link at *the ref* +"###.to_string(); + assert_eq!(parse(source), target); + } + #[test] + fn parses_adam_p_cheatsheet() { + let source = r###"[I'm an inline-style link](https://www.google.com) + +[I'm an inline-style link with title](https://www.google.com "Google's Homepage") + +[I'm a reference-style link][Arbitrary case-insensitive reference text] + +[I'm a relative reference to a repository file](../blob/master/LICENSE) + +[You can use numbers for reference-style link definitions][1] + +Or leave it empty and use the [link text itself][]. + +URLs and URLs in angle brackets will automatically get turned into links. +http://www.example.com or <http://www.example.com> and sometimes +example.com (but not on Github, for example). + +Some text to show that the reference links can follow later. + +[arbitrary case-insensitive reference text]: https://www.mozilla.org +[1]: http://slashdot.org +[link text itself]: http://www.reddit.com"### + .to_string(); + + let target = r###"I'm an inline-style link at *https://www.google.com* + +I'm an inline-style link with title at *https://www.google.com* + +I'm a reference-style link at *https://www.mozilla.org* + +I'm a relative reference to a repository file at *../blob/master/LICENSE* + +You can use numbers for reference-style link definitions at *http://slashdot.org* + +Or leave it empty and use the link text itself at *http://www.reddit.com*. + +URLs and URLs in angle brackets will automatically get turned into links. +http://www.example.com or <http://www.example.com> and sometimes +example.com (but not on Github, for example). + +Some text to show that the reference links can follow later. +"### + .to_string(); + assert_eq!(parse(source), target); + } +} diff --git a/src/doc/book/tools/src/bin/release_listings.rs b/src/doc/book/tools/src/bin/release_listings.rs new file mode 100644 index 000000000..c371d7b30 --- /dev/null +++ b/src/doc/book/tools/src/bin/release_listings.rs @@ -0,0 +1,159 @@ +#[macro_use] +extern crate lazy_static; + +use regex::Regex; +use std::error::Error; +use std::fs; +use std::fs::File; +use std::io::prelude::*; +use std::io::{BufReader, BufWriter}; +use std::path::{Path, PathBuf}; + +fn main() -> Result<(), Box<dyn Error>> { + // Get all listings from the `listings` directory + let listings_dir = Path::new("listings"); + + // Put the results in the `tmp/listings` directory + let out_dir = Path::new("tmp/listings"); + + // Clear out any existing content in `tmp/listings` + if out_dir.is_dir() { + fs::remove_dir_all(out_dir)?; + } + + // Create a new, empty `tmp/listings` directory + fs::create_dir(out_dir)?; + + // For each chapter in the `listings` directory, + for chapter in fs::read_dir(listings_dir)? { + let chapter = chapter?; + let chapter_path = chapter.path(); + + let chapter_name = chapter_path + .file_name() + .expect("Chapter should've had a name"); + + // Create a corresponding chapter dir in `tmp/listings` + let output_chapter_path = out_dir.join(chapter_name); + fs::create_dir(&output_chapter_path)?; + + // For each listing in the chapter directory, + for listing in fs::read_dir(chapter_path)? { + let listing = listing?; + let listing_path = listing.path(); + + let listing_name = listing_path + .file_name() + .expect("Listing should've had a name"); + + // Create a corresponding listing dir in the tmp chapter dir + let output_listing_dir = output_chapter_path.join(listing_name); + fs::create_dir(&output_listing_dir)?; + + // Copy all the cleaned files in the listing to the tmp directory + copy_cleaned_listing_files(listing_path, output_listing_dir)?; + } + } + + // Create a compressed archive of all the listings + let tarfile = File::create("tmp/listings.tar.gz")?; + let encoder = + flate2::write::GzEncoder::new(tarfile, flate2::Compression::default()); + let mut archive = tar::Builder::new(encoder); + archive.append_dir_all("listings", "tmp/listings")?; + + // Assure whoever is running this that the script exiting successfully, and remind them + // where the generated file ends up + println!("Release tarball of listings in tmp/listings.tar.gz"); + + Ok(()) +} + +// Cleaned listings will not contain: +// +// - `target` directories +// - `output.txt` files used to display output in the book +// - `rustfmt-ignore` files used to signal to update-rustc.sh the listing shouldn't be formatted +// - anchor comments or snip comments +// - empty `main` functions in `lib.rs` files used to trick rustdoc +fn copy_cleaned_listing_files( + from: PathBuf, + to: PathBuf, +) -> Result<(), Box<dyn Error>> { + for item in fs::read_dir(from)? { + let item = item?; + let item_path = item.path(); + + let item_name = + item_path.file_name().expect("Item should've had a name"); + let output_item = to.join(item_name); + + if item_path.is_dir() { + // Don't copy `target` directories + if item_name != "target" { + fs::create_dir(&output_item)?; + copy_cleaned_listing_files(item_path, output_item)?; + } + } else { + // Don't copy output files or files that tell update-rustc.sh not to format + if item_name != "output.txt" && item_name != "rustfmt-ignore" { + let item_extension = item_path.extension(); + if item_extension.is_some() && item_extension.unwrap() == "rs" { + copy_cleaned_rust_file( + item_name, + &item_path, + &output_item, + )?; + } else { + // Copy any non-Rust files without modification + fs::copy(item_path, output_item)?; + } + } + } + } + + Ok(()) +} + +lazy_static! { + static ref ANCHOR_OR_SNIP_COMMENTS: Regex = Regex::new( + r"(?x) + //\s*ANCHOR:\s*[\w_-]+ # Remove all anchor comments + | + //\s*ANCHOR_END:\s*[\w_-]+ # Remove all anchor ending comments + | + //\s*--snip-- # Remove all snip comments + " + ) + .unwrap(); +} + +lazy_static! { + static ref EMPTY_MAIN: Regex = Regex::new(r"fn main\(\) \{}").unwrap(); +} + +// Cleaned Rust files will not contain: +// +// - anchor comments or snip comments +// - empty `main` functions in `lib.rs` files used to trick rustdoc +fn copy_cleaned_rust_file( + item_name: &std::ffi::OsStr, + from: &PathBuf, + to: &PathBuf, +) -> Result<(), Box<dyn Error>> { + let from_buf = BufReader::new(File::open(from)?); + let mut to_buf = BufWriter::new(File::create(to)?); + + for line in from_buf.lines() { + let line = line?; + if !ANCHOR_OR_SNIP_COMMENTS.is_match(&line) + && (item_name != "lib.rs" || !EMPTY_MAIN.is_match(&line)) + { + writeln!(&mut to_buf, "{}", line)?; + } + } + + to_buf.flush()?; + + Ok(()) +} diff --git a/src/doc/book/tools/src/bin/remove_hidden_lines.rs b/src/doc/book/tools/src/bin/remove_hidden_lines.rs new file mode 100644 index 000000000..dc3c59357 --- /dev/null +++ b/src/doc/book/tools/src/bin/remove_hidden_lines.rs @@ -0,0 +1,83 @@ +use std::io; +use std::io::prelude::*; + +fn main() { + write_md(remove_hidden_lines(&read_md())); +} + +fn read_md() -> String { + let mut buffer = String::new(); + match io::stdin().read_to_string(&mut buffer) { + Ok(_) => buffer, + Err(error) => panic!("{}", error), + } +} + +fn write_md(output: String) { + print!("{}", output); +} + +fn remove_hidden_lines(input: &str) -> String { + let mut resulting_lines = vec![]; + let mut within_codeblock = false; + + for line in input.lines() { + if line.starts_with("```") { + within_codeblock = !within_codeblock; + } + + if !within_codeblock || (!line.starts_with("# ") && line != "#") { + resulting_lines.push(line) + } + } + + resulting_lines.join("\n") +} + +#[cfg(test)] +mod tests { + use crate::remove_hidden_lines; + + #[test] + fn hidden_line_in_code_block_is_removed() { + let input = r#" +In this listing: + +``` +fn main() { +# secret +} +``` + +you can see that... + "#; + let output = remove_hidden_lines(input); + + let desired_output = r#" +In this listing: + +``` +fn main() { +} +``` + +you can see that... + "#; + + assert_eq!(output, desired_output); + } + + #[test] + fn headings_arent_removed() { + let input = r#" +# Heading 1 + "#; + let output = remove_hidden_lines(input); + + let desired_output = r#" +# Heading 1 + "#; + + assert_eq!(output, desired_output); + } +} diff --git a/src/doc/book/tools/src/bin/remove_links.rs b/src/doc/book/tools/src/bin/remove_links.rs new file mode 100644 index 000000000..b3f78d70a --- /dev/null +++ b/src/doc/book/tools/src/bin/remove_links.rs @@ -0,0 +1,45 @@ +extern crate regex; + +use regex::{Captures, Regex}; +use std::collections::HashSet; +use std::io; +use std::io::Read; + +fn main() { + let mut buffer = String::new(); + if let Err(e) = io::stdin().read_to_string(&mut buffer) { + panic!("{}", e); + } + + let mut refs = HashSet::new(); + + // Capture all links and link references. + let regex = + r"\[([^\]]+)\](?:(?:\[([^\]]+)\])|(?:\([^\)]+\)))(?i)<!--\signore\s-->"; + let link_regex = Regex::new(regex).unwrap(); + let first_pass = link_regex.replace_all(&buffer, |caps: &Captures<'_>| { + // Save the link reference we want to delete. + if let Some(reference) = caps.get(2) { + refs.insert(reference.as_str().to_string()); + } + + // Put the link title back. + caps.get(1).unwrap().as_str().to_string() + }); + + // Search for the references we need to delete. + let ref_regex = Regex::new(r"(?m)^\[([^\]]+)\]:\s.*\n").unwrap(); + let out = ref_regex.replace_all(&first_pass, |caps: &Captures<'_>| { + let capture = caps.get(1).unwrap().to_owned(); + + // Check if we've marked this reference for deletion ... + if refs.contains(capture.as_str()) { + return "".to_string(); + } + + // ... else we put back everything we captured. + caps.get(0).unwrap().as_str().to_string() + }); + + print!("{}", out); +} diff --git a/src/doc/book/tools/src/bin/remove_markup.rs b/src/doc/book/tools/src/bin/remove_markup.rs new file mode 100644 index 000000000..c42e588e7 --- /dev/null +++ b/src/doc/book/tools/src/bin/remove_markup.rs @@ -0,0 +1,53 @@ +extern crate regex; + +use regex::{Captures, Regex}; +use std::io; +use std::io::Read; + +fn main() { + write_md(remove_markup(read_md())); +} + +fn read_md() -> String { + let mut buffer = String::new(); + match io::stdin().read_to_string(&mut buffer) { + Ok(_) => buffer, + Err(error) => panic!("{}", error), + } +} + +fn write_md(output: String) { + print!("{}", output); +} + +fn remove_markup(input: String) -> String { + let filename_regex = + Regex::new(r#"\A<span class="filename">(.*)</span>\z"#).unwrap(); + // Captions sometimes take up multiple lines. + let caption_start_regex = + Regex::new(r#"\A<span class="caption">(.*)\z"#).unwrap(); + let caption_end_regex = Regex::new(r#"(.*)</span>\z"#).unwrap(); + let regexen = vec![filename_regex, caption_start_regex, caption_end_regex]; + + let lines: Vec<_> = input + .lines() + .flat_map(|line| { + // Remove our syntax highlighting and rustdoc markers. + if line.starts_with("```") { + Some(String::from("```")) + // Remove the span around filenames and captions. + } else { + let result = + regexen.iter().fold(line.to_string(), |result, regex| { + regex + .replace_all(&result, |caps: &Captures<'_>| { + caps.get(1).unwrap().as_str().to_string() + }) + .to_string() + }); + Some(result) + } + }) + .collect(); + lines.join("\n") +} diff --git a/src/doc/book/tools/update-editions.sh b/src/doc/book/tools/update-editions.sh new file mode 100755 index 000000000..bd52bc9c8 --- /dev/null +++ b/src/doc/book/tools/update-editions.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -eu + +OLD_EDITION=2018 +NEW_EDITION=2021 + +find listings/** -name "Cargo.toml" -exec sed -i '' "s/edition = \"$OLD_EDITION\"/edition = \"$NEW_EDITION\"/g" '{}' \; diff --git a/src/doc/book/tools/update-rustc.sh b/src/doc/book/tools/update-rustc.sh new file mode 100755 index 000000000..45a0ce4f6 --- /dev/null +++ b/src/doc/book/tools/update-rustc.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +set -eu + +# Build the book before making any changes for comparison of the output. +echo 'Building book into tmp/book-before before updating...' +mdbook build -d tmp/book-before + +# Rustfmt all listings +echo 'Formatting all listings...' +find -s listings -name Cargo.toml -print0 | while IFS= read -r -d '' f; do + dir_to_fmt=$(dirname "$f") + + # There are a handful of listings we don't want to rustfmt and skipping + # doesn't work; those will have a file in their directory that explains why. + if [ ! -f "${dir_to_fmt}/rustfmt-ignore" ]; then + cd "$dir_to_fmt" + cargo fmt --all && true + cd - > /dev/null + fi +done + +# Get listings without anchor comments in tmp by compiling a release listings +# artifact +echo 'Generate listings without anchor comments...' +cargo run --bin release_listings + +root_dir=$(pwd) + +echo 'Regenerating output...' +# For any listings where we show the output, +find -s listings -name output.txt -print0 | while IFS= read -r -d '' f; do + build_directory=$(dirname "$f") + full_build_directory="${root_dir}/${build_directory}" + full_output_path="${full_build_directory}/output.txt" + tmp_build_directory="tmp/${build_directory}" + + cd "$tmp_build_directory" + + # Save the previous compile time; we're going to keep it to minimize diff + # churn + compile_time=$(sed -E -ne 's/.*Finished (dev|test) \[unoptimized \+ debuginfo] target\(s\) in ([0-9.]*).*/\2/p' "${full_output_path}") + + # Save the hash from the first test binary; we're going to keep it to + # minimize diff churn + test_binary_hash=$(sed -E -ne 's@.*Running [^[:space:]]+( [^[:space:]\(\)]+)? \(target/debug/deps/[^-]*-([^\s]*)\)@\2@p' "${full_output_path}" | head -n 1) + + # Act like this is the first time this listing has been built + cargo clean + + # Run the command in the existing output file + cargo_command=$(sed -ne 's/$ \(.*\)/\1/p' "${full_output_path}") + + # Clear the output file of everything except the command + echo "$ ${cargo_command}" > "${full_output_path}" + + # Regenerate the output and append to the output file. Turn some warnings + # off to reduce output noise, and use one test thread to get consistent + # ordering of tests in the output when the command is `cargo test`. + RUSTFLAGS="-A unused_variables -A dead_code" RUST_TEST_THREADS=1 $cargo_command >> "${full_output_path}" 2>&1 || true + + # Set the project file path to the projects directory plus the crate name + # instead of a path to the computer of whoever is running this + sed -i '' -E -e 's@(Compiling|Checking) ([^\)]*) v0.1.0 (.*)@\1 \2 v0.1.0 (file:///projects/\2)@' "${full_output_path}" + + # Restore the previous compile time, if there is one + if [ -n "${compile_time}" ]; then + sed -i '' -E -e "s/Finished (dev|test) \[unoptimized \+ debuginfo] target\(s\) in [0-9.]*/Finished \1 [unoptimized + debuginfo] target(s) in ${compile_time}/" "${full_output_path}" + fi + + # Restore the previous test binary hash, if there is one + if [ -n "${test_binary_hash}" ]; then + replacement='s@Running ([^[:space:]]+)( [^[:space:]\(\)]+)? \(target/debug/deps/([^-]*)-([^\s]*)\)@Running \1\2 (target/debug/deps/\3-' + replacement+="${test_binary_hash}" + replacement+=')@g' + sed -i '' -E -e "${replacement}" "${full_output_path}" + fi + + # Clean again + cargo clean + + cd - > /dev/null +done + +# Build the book after making all the changes +echo 'Building book into tmp/book-after after updating...' +mdbook build -d tmp/book-after + +# Run the megadiff script that removes all files that are the same, leaving only files to audit +echo 'Removing tmp files that had no changes from the update...' +./tools/megadiff.sh + +echo 'Done.' |