diff options
Diffstat (limited to '')
| -rw-r--r-- | .cargo_vcs_info.json | 2 | ||||
| -rw-r--r-- | .github/workflows/ci.yml | 26 | ||||
| -rw-r--r-- | Cargo.toml | 11 | ||||
| -rw-r--r-- | Cargo.toml.orig | 5 | ||||
| -rw-r--r-- | build.rs | 26 | ||||
| -rw-r--r-- | build/probe.rs | 6 | ||||
| -rw-r--r-- | src/extra.rs | 75 | ||||
| -rw-r--r-- | src/fallback.rs | 207 | ||||
| -rw-r--r-- | src/lib.rs | 81 | ||||
| -rw-r--r-- | src/marker.rs | 18 | ||||
| -rw-r--r-- | src/wrapper.rs | 75 | ||||
| -rw-r--r-- | tests/marker.rs | 1 | ||||
| -rw-r--r-- | tests/test.rs | 247 |
13 files changed, 606 insertions, 174 deletions
diff --git a/.cargo_vcs_info.json b/.cargo_vcs_info.json index 0448b81..3c80686 100644 --- a/.cargo_vcs_info.json +++ b/.cargo_vcs_info.json @@ -1,6 +1,6 @@ { "git": { - "sha1": "ea778eb80729a01094b033c45633122052d96fd4" + "sha1": "fd2c998967a65d3378f06f2480abd4d34f4697ab" }, "path_in_vcs": "" }
\ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c51ca9c..2715f2a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: strategy: fail-fast: false matrix: - rust: [1.56.0, stable, beta] + rust: [1.63.0, stable, beta] timeout-minutes: 45 steps: - uses: actions/checkout@v4 @@ -78,6 +78,29 @@ jobs: env: RUSTFLAGS: -Z allow-features= --cfg procmacro2_backtrace ${{env.RUSTFLAGS}} + msrv: + name: Rust 1.56.0 + needs: pre_ci + if: needs.pre_ci.outputs.continue + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@1.56.0 + with: + components: rust-src + - run: cargo check + - run: cargo check --no-default-features + - run: cargo check --features span-locations + - name: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo check + run: cargo check + env: + RUSTFLAGS: --cfg procmacro2_semver_exempt ${{env.RUSTFLAGS}} + - name: RUSTFLAGS='--cfg procmacro2_semver_exempt' cargo check --no-default-features + run: cargo check --no-default-features + env: + RUSTFLAGS: --cfg procmacro2_semver_exempt ${{env.RUSTFLAGS}} + minimal: name: Minimal versions needs: pre_ci @@ -175,6 +198,7 @@ jobs: timeout-minutes: 45 steps: - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable - uses: dtolnay/install@cargo-outdated - run: cargo outdated --workspace --exit-code 1 - run: cargo outdated --manifest-path fuzz/Cargo.toml --exit-code 1 @@ -13,7 +13,7 @@ edition = "2021" rust-version = "1.56" name = "proc-macro2" -version = "1.0.76" +version = "1.0.81" authors = [ "David Tolnay <dtolnay@gmail.com>", "Alex Crichton <alex@alexcrichton.com>", @@ -53,13 +53,22 @@ doc-scrape-examples = false [dependencies.unicode-ident] version = "1.0" +[dev-dependencies.flate2] +version = "1.0" + [dev-dependencies.quote] version = "1.0" default_features = false +[dev-dependencies.rayon] +version = "1.0" + [dev-dependencies.rustversion] version = "1" +[dev-dependencies.tar] +version = "0.4" + [features] default = ["proc-macro"] nightly = [] diff --git a/Cargo.toml.orig b/Cargo.toml.orig index 6f7c786..f543d9b 100644 --- a/Cargo.toml.orig +++ b/Cargo.toml.orig @@ -1,6 +1,6 @@ [package] name = "proc-macro2" -version = "1.0.76" +version = "1.0.81" authors = ["David Tolnay <dtolnay@gmail.com>", "Alex Crichton <alex@alexcrichton.com>"] autobenches = false categories = ["development-tools::procedural-macro-helpers"] @@ -24,8 +24,11 @@ features = ["span-locations"] unicode-ident = "1.0" [dev-dependencies] +flate2 = "1.0" quote = { version = "1.0", default_features = false } +rayon = "1.0" rustversion = "1" +tar = "0.4" [features] proc-macro = [] @@ -36,10 +36,10 @@ use std::env; use std::ffi::OsString; +use std::iter; use std::path::Path; use std::process::{self, Command, Stdio}; use std::str; -use std::u32; fn main() { let rustc = rustc_minor_version().unwrap_or(u32::MAX); @@ -63,6 +63,11 @@ fn main() { println!("cargo:rustc-cfg=no_source_text"); } + if rustc < 79 { + println!("cargo:rustc-cfg=no_literal_byte_character"); + println!("cargo:rustc-cfg=no_literal_c_string"); + } + if !cfg!(feature = "proc-macro") { println!("cargo:rerun-if-changed=build.rs"); return; @@ -138,15 +143,15 @@ fn compile_probe(rustc_bootstrap: bool) -> bool { let out_dir = cargo_env_var("OUT_DIR"); let probefile = Path::new("build").join("probe.rs"); - // Make sure to pick up Cargo rustc configuration. - let mut cmd = if let Some(wrapper) = env::var_os("RUSTC_WRAPPER") { - let mut cmd = Command::new(wrapper); - // The wrapper's first argument is supposed to be the path to rustc. - cmd.arg(rustc); - cmd - } else { - Command::new(rustc) - }; + let rustc_wrapper = env::var_os("RUSTC_WRAPPER").filter(|wrapper| !wrapper.is_empty()); + let rustc_workspace_wrapper = + env::var_os("RUSTC_WORKSPACE_WRAPPER").filter(|wrapper| !wrapper.is_empty()); + let mut rustc = rustc_wrapper + .into_iter() + .chain(rustc_workspace_wrapper) + .chain(iter::once(rustc)); + let mut cmd = Command::new(rustc.next().unwrap()); + cmd.args(rustc); if !rustc_bootstrap { cmd.env_remove("RUSTC_BOOTSTRAP"); @@ -156,6 +161,7 @@ fn compile_probe(rustc_bootstrap: bool) -> bool { .arg("--edition=2021") .arg("--crate-name=proc_macro2") .arg("--crate-type=lib") + .arg("--cap-lints=allow") .arg("--emit=dep-info,metadata") .arg("--out-dir") .arg(out_dir) diff --git a/build/probe.rs b/build/probe.rs index 5afa13a..2c4947a 100644 --- a/build/probe.rs +++ b/build/probe.rs @@ -6,9 +6,13 @@ extern crate proc_macro; -use core::ops::RangeBounds; +use core::ops::{Range, RangeBounds}; use proc_macro::{Literal, Span}; +pub fn byte_range(this: &Span) -> Range<usize> { + this.byte_range() +} + pub fn join(this: &Span, other: Span) -> Option<Span> { this.join(other) } diff --git a/src/extra.rs b/src/extra.rs index 4a69d46..543ec1d 100644 --- a/src/extra.rs +++ b/src/extra.rs @@ -3,18 +3,85 @@ use crate::fallback; use crate::imp; -use crate::marker::Marker; +use crate::marker::{ProcMacroAutoTraits, MARKER}; use crate::Span; use core::fmt::{self, Debug}; +/// Invalidate any `proc_macro2::Span` that exist on the current thread. +/// +/// The implementation of `Span` uses thread-local data structures and this +/// function clears them. Calling any method on a `Span` on the current thread +/// created prior to the invalidation will return incorrect values or crash. +/// +/// This function is useful for programs that process more than 2<sup>32</sup> +/// bytes of Rust source code on the same thread. Just like rustc, proc-macro2 +/// uses 32-bit source locations, and these wrap around when the total source +/// code processed by the same thread exceeds 2<sup>32</sup> bytes (4 +/// gigabytes). After a wraparound, `Span` methods such as `source_text()` can +/// return wrong data. +/// +/// # Example +/// +/// As of late 2023, there is 200 GB of Rust code published on crates.io. +/// Looking at just the newest version of every crate, it is 16 GB of code. So a +/// workload that involves parsing it all would overflow a 32-bit source +/// location unless spans are being invalidated. +/// +/// ``` +/// use flate2::read::GzDecoder; +/// use std::ffi::OsStr; +/// use std::io::{BufReader, Read}; +/// use std::str::FromStr; +/// use tar::Archive; +/// +/// rayon::scope(|s| { +/// for krate in every_version_of_every_crate() { +/// s.spawn(move |_| { +/// proc_macro2::extra::invalidate_current_thread_spans(); +/// +/// let reader = BufReader::new(krate); +/// let tar = GzDecoder::new(reader); +/// let mut archive = Archive::new(tar); +/// for entry in archive.entries().unwrap() { +/// let mut entry = entry.unwrap(); +/// let path = entry.path().unwrap(); +/// if path.extension() != Some(OsStr::new("rs")) { +/// continue; +/// } +/// let mut content = String::new(); +/// entry.read_to_string(&mut content).unwrap(); +/// match proc_macro2::TokenStream::from_str(&content) { +/// Ok(tokens) => {/* ... */}, +/// Err(_) => continue, +/// } +/// } +/// }); +/// } +/// }); +/// # +/// # fn every_version_of_every_crate() -> Vec<std::fs::File> { +/// # Vec::new() +/// # } +/// ``` +/// +/// # Panics +/// +/// This function is not applicable to and will panic if called from a +/// procedural macro. +#[cfg(span_locations)] +#[cfg_attr(doc_cfg, doc(cfg(feature = "span-locations")))] +pub fn invalidate_current_thread_spans() { + crate::imp::invalidate_current_thread_spans(); +} + /// An object that holds a [`Group`]'s `span_open()` and `span_close()` together -/// (in a more compact representation than holding those 2 spans individually. +/// in a more compact representation than holding those 2 spans individually. /// /// [`Group`]: crate::Group #[derive(Copy, Clone)] pub struct DelimSpan { inner: DelimSpanEnum, - _marker: Marker, + _marker: ProcMacroAutoTraits, } #[derive(Copy, Clone)] @@ -45,7 +112,7 @@ impl DelimSpan { DelimSpan { inner, - _marker: Marker, + _marker: MARKER, } } diff --git a/src/fallback.rs b/src/fallback.rs index 7b40427..b42537b 100644 --- a/src/fallback.rs +++ b/src/fallback.rs @@ -11,9 +11,13 @@ use core::cell::RefCell; use core::cmp; use core::fmt::{self, Debug, Display, Write}; use core::mem::ManuallyDrop; +#[cfg(span_locations)] +use core::ops::Range; use core::ops::RangeBounds; use core::ptr; -use core::str::FromStr; +use core::str::{self, FromStr}; +use std::ffi::CStr; +#[cfg(procmacro2_semver_exempt)] use std::path::PathBuf; /// Force use of proc-macro2's fallback implementation of the API for now, even @@ -151,9 +155,9 @@ fn get_cursor(src: &str) -> Cursor { // Create a dummy file & add it to the source map #[cfg(not(fuzzing))] - SOURCE_MAP.with(|cm| { - let mut cm = cm.borrow_mut(); - let span = cm.add_file(src); + SOURCE_MAP.with(|sm| { + let mut sm = sm.borrow_mut(); + let span = sm.add_file(src); Cursor { rest: src, off: span.lo, @@ -295,11 +299,13 @@ impl IntoIterator for TokenStream { } } +#[cfg(procmacro2_semver_exempt)] #[derive(Clone, PartialEq, Eq)] pub(crate) struct SourceFile { path: PathBuf, } +#[cfg(procmacro2_semver_exempt)] impl SourceFile { /// Get the path to this source file as a string. pub fn path(&self) -> PathBuf { @@ -311,6 +317,7 @@ impl SourceFile { } } +#[cfg(procmacro2_semver_exempt)] impl Debug for SourceFile { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("SourceFile") @@ -334,6 +341,12 @@ thread_local! { }); } +#[cfg(span_locations)] +pub(crate) fn invalidate_current_thread_spans() { + #[cfg(not(fuzzing))] + SOURCE_MAP.with(|sm| sm.borrow_mut().files.truncate(1)); +} + #[cfg(all(span_locations, not(fuzzing)))] struct FileInfo { source_text: String, @@ -366,7 +379,7 @@ impl FileInfo { span.lo >= self.span.lo && span.hi <= self.span.hi } - fn source_text(&mut self, span: Span) -> String { + fn byte_range(&mut self, span: Span) -> Range<usize> { let lo_char = (span.lo - self.span.lo) as usize; // Look up offset of the largest already-computed char index that is @@ -395,11 +408,15 @@ impl FileInfo { let trunc_lo = &self.source_text[lo_byte..]; let char_len = (span.hi - span.lo) as usize; - let source_text = match trunc_lo.char_indices().nth(char_len) { - Some((offset, _ch)) => &trunc_lo[..offset], - None => trunc_lo, - }; - source_text.to_owned() + lo_byte..match trunc_lo.char_indices().nth(char_len) { + Some((offset, _ch)) => lo_byte + offset, + None => self.source_text.len(), + } + } + + fn source_text(&mut self, span: Span) -> String { + let byte_range = self.byte_range(span); + self.source_text[byte_range].to_owned() } } @@ -534,22 +551,37 @@ impl Span { }; #[cfg(not(fuzzing))] - SOURCE_MAP.with(|cm| { - let cm = cm.borrow(); - let path = cm.filepath(*self); + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + let path = sm.filepath(*self); SourceFile { path } }) } #[cfg(span_locations)] + pub fn byte_range(&self) -> Range<usize> { + #[cfg(fuzzing)] + return 0..0; + + #[cfg(not(fuzzing))] + { + if self.is_call_site() { + 0..0 + } else { + SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).byte_range(*self)) + } + } + } + + #[cfg(span_locations)] pub fn start(&self) -> LineColumn { #[cfg(fuzzing)] return LineColumn { line: 0, column: 0 }; #[cfg(not(fuzzing))] - SOURCE_MAP.with(|cm| { - let cm = cm.borrow(); - let fi = cm.fileinfo(*self); + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + let fi = sm.fileinfo(*self); fi.offset_line_column(self.lo as usize) }) } @@ -560,9 +592,9 @@ impl Span { return LineColumn { line: 0, column: 0 }; #[cfg(not(fuzzing))] - SOURCE_MAP.with(|cm| { - let cm = cm.borrow(); - let fi = cm.fileinfo(*self); + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); + let fi = sm.fileinfo(*self); fi.offset_line_column(self.hi as usize) }) } @@ -581,10 +613,10 @@ impl Span { }; #[cfg(not(fuzzing))] - SOURCE_MAP.with(|cm| { - let cm = cm.borrow(); + SOURCE_MAP.with(|sm| { + let sm = sm.borrow(); // If `other` is not within the same FileInfo as us, return None. - if !cm.fileinfo(*self).span_within(other) { + if !sm.fileinfo(*self).span_within(other) { return None; } Some(Span { @@ -609,7 +641,7 @@ impl Span { if self.is_call_site() { None } else { - Some(SOURCE_MAP.with(|cm| cm.borrow_mut().fileinfo_mut(*self).source_text(*self))) + Some(SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).source_text(*self))) } } } @@ -895,7 +927,7 @@ impl Debug for Ident { #[derive(Clone)] pub(crate) struct Literal { - repr: String, + pub(crate) repr: String, span: Span, } @@ -976,71 +1008,98 @@ impl Literal { Literal::_new(s) } - pub fn string(t: &str) -> Literal { - let mut repr = String::with_capacity(t.len() + 2); + pub fn string(string: &str) -> Literal { + let mut repr = String::with_capacity(string.len() + 2); repr.push('"'); - let mut chars = t.chars(); - while let Some(ch) = chars.next() { - if ch == '\0' { - repr.push_str( - if chars - .as_str() - .starts_with(|next| '0' <= next && next <= '7') - { - // circumvent clippy::octal_escapes lint - "\\x00" - } else { - "\\0" - }, - ); - } else if ch == '\'' { - // escape_debug turns this into "\'" which is unnecessary. - repr.push(ch); - } else { - repr.extend(ch.escape_debug()); - } - } + escape_utf8(string, &mut repr); repr.push('"'); Literal::_new(repr) } - pub fn character(t: char) -> Literal { + pub fn character(ch: char) -> Literal { let mut repr = String::new(); repr.push('\''); - if t == '"' { + if ch == '"' { // escape_debug turns this into '\"' which is unnecessary. - repr.push(t); + repr.push(ch); } else { - repr.extend(t.escape_debug()); + repr.extend(ch.escape_debug()); + } + repr.push('\''); + Literal::_new(repr) + } + + pub fn byte_character(byte: u8) -> Literal { + let mut repr = "b'".to_string(); + #[allow(clippy::match_overlapping_arm)] + match byte { + b'\0' => repr.push_str(r"\0"), + b'\t' => repr.push_str(r"\t"), + b'\n' => repr.push_str(r"\n"), + b'\r' => repr.push_str(r"\r"), + b'\'' => repr.push_str(r"\'"), + b'\\' => repr.push_str(r"\\"), + b'\x20'..=b'\x7E' => repr.push(byte as char), + _ => { + let _ = write!(repr, r"\x{:02X}", byte); + } } repr.push('\''); Literal::_new(repr) } pub fn byte_string(bytes: &[u8]) -> Literal { - let mut escaped = "b\"".to_string(); + let mut repr = "b\"".to_string(); let mut bytes = bytes.iter(); while let Some(&b) = bytes.next() { #[allow(clippy::match_overlapping_arm)] match b { - b'\0' => escaped.push_str(match bytes.as_slice().first() { + b'\0' => repr.push_str(match bytes.as_slice().first() { // circumvent clippy::octal_escapes lint Some(b'0'..=b'7') => r"\x00", _ => r"\0", }), - b'\t' => escaped.push_str(r"\t"), - b'\n' => escaped.push_str(r"\n"), - b'\r' => escaped.push_str(r"\r"), - b'"' => escaped.push_str("\\\""), - b'\\' => escaped.push_str("\\\\"), - b'\x20'..=b'\x7E' => escaped.push(b as char), + b'\t' => repr.push_str(r"\t"), + b'\n' => repr.push_str(r"\n"), + b'\r' => repr.push_str(r"\r"), + b'"' => repr.push_str("\\\""), + b'\\' => repr.push_str(r"\\"), + b'\x20'..=b'\x7E' => repr.push(b as char), _ => { - let _ = write!(escaped, "\\x{:02X}", b); + let _ = write!(repr, r"\x{:02X}", b); + } + } + } + repr.push('"'); + Literal::_new(repr) + } + + pub fn c_string(string: &CStr) -> Literal { + let mut repr = "c\"".to_string(); + let mut bytes = string.to_bytes(); + while !bytes.is_empty() { + let (valid, invalid) = match str::from_utf8(bytes) { + Ok(all_valid) => { + bytes = b""; + (all_valid, bytes) + } + Err(utf8_error) => { + let (valid, rest) = bytes.split_at(utf8_error.valid_up_to()); + let valid = str::from_utf8(valid).unwrap(); + let invalid = utf8_error + .error_len() + .map_or(rest, |error_len| &rest[..error_len]); + bytes = &bytes[valid.len() + invalid.len()..]; + (valid, invalid) } + }; + escape_utf8(valid, &mut repr); + for &byte in invalid { + let _ = write!(repr, r"\x{:02X}", byte); } } - escaped.push('"'); - Literal::_new(escaped) + repr.push('"'); + Literal::_new(repr) } pub fn span(&self) -> Span { @@ -1141,3 +1200,27 @@ impl Debug for Literal { debug.finish() } } + +fn escape_utf8(string: &str, repr: &mut String) { + let mut chars = string.chars(); + while let Some(ch) = chars.next() { + if ch == '\0' { + repr.push_str( + if chars + .as_str() + .starts_with(|next| '0' <= next && next <= '7') + { + // circumvent clippy::octal_escapes lint + r"\x00" + } else { + r"\0" + }, + ); + } else if ch == '\'' { + // escape_debug turns this into "\'" which is unnecessary. + repr.push(ch); + } else { + repr.extend(ch.escape_debug()); + } + } +} @@ -86,7 +86,7 @@ //! a different thread. // Proc-macro2 types in rustdoc of other crates get linked to here. -#![doc(html_root_url = "https://docs.rs/proc-macro2/1.0.76")] +#![doc(html_root_url = "https://docs.rs/proc-macro2/1.0.81")] #![cfg_attr(any(proc_macro_span, super_unstable), feature(proc_macro_span))] #![cfg_attr(super_unstable, feature(proc_macro_def_site))] #![cfg_attr(doc_cfg, feature(doc_cfg))] @@ -96,6 +96,7 @@ clippy::cast_possible_truncation, clippy::checked_conversions, clippy::doc_markdown, + clippy::incompatible_msrv, clippy::items_after_statements, clippy::iter_without_into_iter, clippy::let_underscore_untyped, @@ -160,13 +161,16 @@ mod imp; mod location; use crate::extra::DelimSpan; -use crate::marker::Marker; +use crate::marker::{ProcMacroAutoTraits, MARKER}; use core::cmp::Ordering; use core::fmt::{self, Debug, Display}; use core::hash::{Hash, Hasher}; +#[cfg(span_locations)] +use core::ops::Range; use core::ops::RangeBounds; use core::str::FromStr; use std::error::Error; +use std::ffi::CStr; #[cfg(procmacro2_semver_exempt)] use std::path::PathBuf; @@ -184,27 +188,27 @@ pub use crate::location::LineColumn; #[derive(Clone)] pub struct TokenStream { inner: imp::TokenStream, - _marker: Marker, + _marker: ProcMacroAutoTraits, } /// Error returned from `TokenStream::from_str`. pub struct LexError { inner: imp::LexError, - _marker: Marker, + _marker: ProcMacroAutoTraits, } impl TokenStream { fn _new(inner: imp::TokenStream) -> Self { TokenStream { inner, - _marker: Marker, + _marker: MARKER, } } fn _new_fallback(inner: fallback::TokenStream) -> Self { TokenStream { inner: inner.into(), - _marker: Marker, + _marker: MARKER, } } @@ -241,7 +245,7 @@ impl FromStr for TokenStream { fn from_str(src: &str) -> Result<TokenStream, LexError> { let e = src.parse().map_err(|e| LexError { inner: e, - _marker: Marker, + _marker: MARKER, })?; Ok(TokenStream::_new(e)) } @@ -339,7 +343,7 @@ impl Error for LexError {} #[derive(Clone, PartialEq, Eq)] pub struct SourceFile { inner: imp::SourceFile, - _marker: Marker, + _marker: ProcMacroAutoTraits, } #[cfg(all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)))] @@ -347,7 +351,7 @@ impl SourceFile { fn _new(inner: imp::SourceFile) -> Self { SourceFile { inner, - _marker: Marker, + _marker: MARKER, } } @@ -386,21 +390,21 @@ impl Debug for SourceFile { #[derive(Copy, Clone)] pub struct Span { inner: imp::Span, - _marker: Marker, + _marker: ProcMacroAutoTraits, } impl Span { fn _new(inner: imp::Span) -> Self { Span { inner, - _marker: Marker, + _marker: MARKER, } } fn _new_fallback(inner: fallback::Span) -> Self { Span { inner: inner.into(), - _marker: Marker, + _marker: MARKER, } } @@ -472,6 +476,21 @@ impl Span { SourceFile::_new(self.inner.source_file()) } + /// Returns the span's byte position range in the source file. + /// + /// This method requires the `"span-locations"` feature to be enabled. + /// + /// When executing in a procedural macro context, the returned range is only + /// accurate if compiled with a nightly toolchain. The stable toolchain does + /// not have this information available. When executing outside of a + /// procedural macro, such as main.rs or build.rs, the byte range is always + /// accurate regardless of toolchain. + #[cfg(span_locations)] + #[cfg_attr(doc_cfg, doc(cfg(feature = "span-locations")))] + pub fn byte_range(&self) -> Range<usize> { + self.inner.byte_range() + } + /// Get the starting line/column in the source file for this span. /// /// This method requires the `"span-locations"` feature to be enabled. @@ -659,12 +678,12 @@ pub enum Delimiter { Brace, /// `[ ... ]` Bracket, - /// `Ø ... Ø` + /// `∅ ... ∅` /// - /// An implicit delimiter, that may, for example, appear around tokens + /// An invisible delimiter, that may, for example, appear around tokens /// coming from a "macro variable" `$var`. It is important to preserve /// operator priorities in cases like `$var * 3` where `$var` is `1 + 2`. - /// Implicit delimiters may not survive roundtrip of a token stream through + /// Invisible delimiters may not survive roundtrip of a token stream through /// a string. None, } @@ -919,14 +938,14 @@ impl Debug for Punct { #[derive(Clone)] pub struct Ident { inner: imp::Ident, - _marker: Marker, + _marker: ProcMacroAutoTraits, } impl Ident { fn _new(inner: imp::Ident) -> Self { Ident { inner, - _marker: Marker, + _marker: MARKER, } } @@ -1046,7 +1065,7 @@ impl Debug for Ident { #[derive(Clone)] pub struct Literal { inner: imp::Literal, - _marker: Marker, + _marker: ProcMacroAutoTraits, } macro_rules! suffixed_int_literals { @@ -1093,14 +1112,14 @@ impl Literal { fn _new(inner: imp::Literal) -> Self { Literal { inner, - _marker: Marker, + _marker: MARKER, } } fn _new_fallback(inner: fallback::Literal) -> Self { Literal { inner: inner.into(), - _marker: Marker, + _marker: MARKER, } } @@ -1216,9 +1235,19 @@ impl Literal { Literal::_new(imp::Literal::character(ch)) } + /// Byte character literal. + pub fn byte_character(byte: u8) -> Literal { + Literal::_new(imp::Literal::byte_character(byte)) + } + /// Byte string literal. - pub fn byte_string(s: &[u8]) -> Literal { - Literal::_new(imp::Literal::byte_string(s)) + pub fn byte_string(bytes: &[u8]) -> Literal { + Literal::_new(imp::Literal::byte_string(bytes)) + } + + /// C string literal. + pub fn c_string(string: &CStr) -> Literal { + Literal::_new(imp::Literal::c_string(string)) } /// Returns the span encompassing this literal. @@ -1260,7 +1289,7 @@ impl FromStr for Literal { fn from_str(repr: &str) -> Result<Self, LexError> { repr.parse().map(Literal::_new).map_err(|inner| LexError { inner, - _marker: Marker, + _marker: MARKER, }) } } @@ -1279,7 +1308,7 @@ impl Display for Literal { /// Public implementation details for the `TokenStream` type, such as iterators. pub mod token_stream { - use crate::marker::Marker; + use crate::marker::{ProcMacroAutoTraits, MARKER}; use crate::{imp, TokenTree}; use core::fmt::{self, Debug}; @@ -1292,7 +1321,7 @@ pub mod token_stream { #[derive(Clone)] pub struct IntoIter { inner: imp::TokenTreeIter, - _marker: Marker, + _marker: ProcMacroAutoTraits, } impl Iterator for IntoIter { @@ -1321,7 +1350,7 @@ pub mod token_stream { fn into_iter(self) -> IntoIter { IntoIter { inner: self.inner.into_iter(), - _marker: Marker, + _marker: MARKER, } } } diff --git a/src/marker.rs b/src/marker.rs index e8874bd..23b94ce 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -4,18 +4,14 @@ use core::panic::{RefUnwindSafe, UnwindSafe}; // Zero sized marker with the correct set of autotrait impls we want all proc // macro types to have. -pub(crate) type Marker = PhantomData<ProcMacroAutoTraits>; +#[derive(Copy, Clone)] +#[cfg_attr( + all(procmacro2_semver_exempt, any(not(wrap_proc_macro), super_unstable)), + derive(PartialEq, Eq) +)] +pub(crate) struct ProcMacroAutoTraits(PhantomData<Rc<()>>); -pub(crate) use self::value::*; - -mod value { - pub(crate) use core::marker::PhantomData as Marker; -} - -pub(crate) struct ProcMacroAutoTraits( - #[allow(dead_code)] // https://github.com/rust-lang/rust/issues/119645 - Rc<()>, -); +pub(crate) const MARKER: ProcMacroAutoTraits = ProcMacroAutoTraits(PhantomData); impl UnwindSafe for ProcMacroAutoTraits {} impl RefUnwindSafe for ProcMacroAutoTraits {} diff --git a/src/wrapper.rs b/src/wrapper.rs index f5eb826..87e348d 100644 --- a/src/wrapper.rs +++ b/src/wrapper.rs @@ -3,8 +3,11 @@ use crate::detection::inside_proc_macro; use crate::location::LineColumn; use crate::{fallback, Delimiter, Punct, Spacing, TokenTree}; use core::fmt::{self, Debug, Display}; +#[cfg(span_locations)] +use core::ops::Range; use core::ops::RangeBounds; use core::str::FromStr; +use std::ffi::CStr; use std::panic; #[cfg(super_unstable)] use std::path::PathBuf; @@ -461,6 +464,17 @@ impl Span { } #[cfg(span_locations)] + pub fn byte_range(&self) -> Range<usize> { + match self { + #[cfg(proc_macro_span)] + Span::Compiler(s) => s.byte_range(), + #[cfg(not(proc_macro_span))] + Span::Compiler(_) => 0..0, + Span::Fallback(s) => s.byte_range(), + } + } + + #[cfg(span_locations)] pub fn start(&self) -> LineColumn { match self { Span::Compiler(_) => LineColumn { line: 0, column: 0 }, @@ -833,19 +847,38 @@ impl Literal { } } - pub fn string(t: &str) -> Literal { + pub fn string(string: &str) -> Literal { if inside_proc_macro() { - Literal::Compiler(proc_macro::Literal::string(t)) + Literal::Compiler(proc_macro::Literal::string(string)) } else { - Literal::Fallback(fallback::Literal::string(t)) + Literal::Fallback(fallback::Literal::string(string)) } } - pub fn character(t: char) -> Literal { + pub fn character(ch: char) -> Literal { if inside_proc_macro() { - Literal::Compiler(proc_macro::Literal::character(t)) + Literal::Compiler(proc_macro::Literal::character(ch)) + } else { + Literal::Fallback(fallback::Literal::character(ch)) + } + } + + pub fn byte_character(byte: u8) -> Literal { + if inside_proc_macro() { + Literal::Compiler({ + #[cfg(not(no_literal_byte_character))] + { + proc_macro::Literal::byte_character(byte) + } + + #[cfg(no_literal_byte_character)] + { + let fallback = fallback::Literal::byte_character(byte); + fallback.repr.parse::<proc_macro::Literal>().unwrap() + } + }) } else { - Literal::Fallback(fallback::Literal::character(t)) + Literal::Fallback(fallback::Literal::byte_character(byte)) } } @@ -857,6 +890,25 @@ impl Literal { } } + pub fn c_string(string: &CStr) -> Literal { + if inside_proc_macro() { + Literal::Compiler({ + #[cfg(not(no_literal_c_string))] + { + proc_macro::Literal::c_string(string) + } + + #[cfg(no_literal_c_string)] + { + let fallback = fallback::Literal::c_string(string); + fallback.repr.parse::<proc_macro::Literal>().unwrap() + } + }) + } else { + Literal::Fallback(fallback::Literal::c_string(string)) + } + } + pub fn span(&self) -> Span { match self { Literal::Compiler(lit) => Span::Compiler(lit.span()), @@ -928,3 +980,14 @@ impl Debug for Literal { } } } + +#[cfg(span_locations)] +pub(crate) fn invalidate_current_thread_spans() { + if inside_proc_macro() { + panic!( + "proc_macro2::extra::invalidate_current_thread_spans is not available in procedural macros" + ); + } else { + crate::fallback::invalidate_current_thread_spans(); + } +} diff --git a/tests/marker.rs b/tests/marker.rs index d08fbfc..99f64c0 100644 --- a/tests/marker.rs +++ b/tests/marker.rs @@ -21,6 +21,7 @@ macro_rules! assert_impl { $( { // Implemented for types that implement $marker. + #[allow(dead_code)] trait IsNotImplemented { fn assert_not_implemented() {} } diff --git a/tests/test.rs b/tests/test.rs index b75cd55..3923d33 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -1,11 +1,14 @@ #![allow( clippy::assertions_on_result_states, clippy::items_after_statements, + clippy::needless_pass_by_value, + clippy::needless_raw_string_hashes, clippy::non_ascii_literal, clippy::octal_escapes )] use proc_macro2::{Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; +use std::ffi::CStr; use std::iter; use std::str::{self, FromStr}; @@ -96,12 +99,22 @@ fn lifetime_invalid() { #[test] fn literal_string() { - assert_eq!(Literal::string("foo").to_string(), "\"foo\""); - assert_eq!(Literal::string("\"").to_string(), "\"\\\"\""); - assert_eq!(Literal::string("didn't").to_string(), "\"didn't\""); - assert_eq!( - Literal::string("a\00b\07c\08d\0e\0").to_string(), - "\"a\\x000b\\x007c\\08d\\0e\\0\"", + #[track_caller] + fn assert(literal: Literal, expected: &str) { + assert_eq!(literal.to_string(), expected.trim()); + } + + assert(Literal::string(""), r#" "" "#); + assert(Literal::string("aA"), r#" "aA" "#); + assert(Literal::string("\t"), r#" "\t" "#); + assert(Literal::string("❤"), r#" "❤" "#); + assert(Literal::string("'"), r#" "'" "#); + assert(Literal::string("\""), r#" "\"" "#); + assert(Literal::string("\0"), r#" "\0" "#); + assert(Literal::string("\u{1}"), r#" "\u{1}" "#); + assert( + Literal::string("a\00b\07c\08d\0e\0"), + r#" "a\x000b\x007c\08d\0e\0" "#, ); "\"\\\r\n x\"".parse::<TokenStream>().unwrap(); @@ -133,15 +146,42 @@ fn literal_raw_string() { } #[test] +fn literal_byte_character() { + #[track_caller] + fn assert(literal: Literal, expected: &str) { + assert_eq!(literal.to_string(), expected.trim()); + } + + assert(Literal::byte_character(b'a'), r#" b'a' "#); + assert(Literal::byte_character(b'\0'), r#" b'\0' "#); + assert(Literal::byte_character(b'\t'), r#" b'\t' "#); + assert(Literal::byte_character(b'\n'), r#" b'\n' "#); + assert(Literal::byte_character(b'\r'), r#" b'\r' "#); + assert(Literal::byte_character(b'\''), r#" b'\'' "#); + assert(Literal::byte_character(b'\\'), r#" b'\\' "#); + assert(Literal::byte_character(b'\x1f'), r#" b'\x1F' "#); + assert(Literal::byte_character(b'"'), r#" b'"' "#); +} + +#[test] fn literal_byte_string() { - assert_eq!(Literal::byte_string(b"").to_string(), "b\"\""); - assert_eq!( - Literal::byte_string(b"\0\t\n\r\"\\2\x10").to_string(), - "b\"\\0\\t\\n\\r\\\"\\\\2\\x10\"", - ); - assert_eq!( - Literal::byte_string(b"a\00b\07c\08d\0e\0").to_string(), - "b\"a\\x000b\\x007c\\08d\\0e\\0\"", + #[track_caller] + fn assert(literal: Literal, expected: &str) { + assert_eq!(literal.to_string(), expected.trim()); + } + + assert(Literal::byte_string(b""), r#" b"" "#); + assert(Literal::byte_string(b"\0"), r#" b"\0" "#); + assert(Literal::byte_string(b"\t"), r#" b"\t" "#); + assert(Literal::byte_string(b"\n"), r#" b"\n" "#); + assert(Literal::byte_string(b"\r"), r#" b"\r" "#); + assert(Literal::byte_string(b"\""), r#" b"\"" "#); + assert(Literal::byte_string(b"\\"), r#" b"\\" "#); + assert(Literal::byte_string(b"\x1f"), r#" b"\x1F" "#); + assert(Literal::byte_string(b"'"), r#" b"'" "#); + assert( + Literal::byte_string(b"a\00b\07c\08d\0e\0"), + r#" b"a\x000b\x007c\08d\0e\0" "#, ); "b\"\\\r\n x\"".parse::<TokenStream>().unwrap(); @@ -152,6 +192,41 @@ fn literal_byte_string() { #[test] fn literal_c_string() { + #[track_caller] + fn assert(literal: Literal, expected: &str) { + assert_eq!(literal.to_string(), expected.trim()); + } + + assert(Literal::c_string(<&CStr>::default()), r#" c"" "#); + assert( + Literal::c_string(CStr::from_bytes_with_nul(b"aA\0").unwrap()), + r#" c"aA" "#, + ); + assert( + Literal::c_string(CStr::from_bytes_with_nul(b"aA\0").unwrap()), + r#" c"aA" "#, + ); + assert( + Literal::c_string(CStr::from_bytes_with_nul(b"\t\0").unwrap()), + r#" c"\t" "#, + ); + assert( + Literal::c_string(CStr::from_bytes_with_nul(b"\xE2\x9D\xA4\0").unwrap()), + r#" c"❤" "#, + ); + assert( + Literal::c_string(CStr::from_bytes_with_nul(b"'\0").unwrap()), + r#" c"'" "#, + ); + assert( + Literal::c_string(CStr::from_bytes_with_nul(b"\"\0").unwrap()), + r#" c"\"" "#, + ); + assert( + Literal::c_string(CStr::from_bytes_with_nul(b"\x7F\xFF\xFE\xCC\xB3\0").unwrap()), + r#" c"\u{7f}\xFF\xFE\u{333}" "#, + ); + let strings = r###" c"hello\x80我叫\u{1F980}" // from the RFC cr"\" @@ -188,49 +263,80 @@ fn literal_c_string() { #[test] fn literal_character() { - assert_eq!(Literal::character('x').to_string(), "'x'"); - assert_eq!(Literal::character('\'').to_string(), "'\\''"); - assert_eq!(Literal::character('"').to_string(), "'\"'"); + #[track_caller] + fn assert(literal: Literal, expected: &str) { + assert_eq!(literal.to_string(), expected.trim()); + } + + assert(Literal::character('a'), r#" 'a' "#); + assert(Literal::character('\t'), r#" '\t' "#); + assert(Literal::character('❤'), r#" '❤' "#); + assert(Literal::character('\''), r#" '\'' "#); + assert(Literal::character('"'), r#" '"' "#); + assert(Literal::character('\0'), r#" '\0' "#); + assert(Literal::character('\u{1}'), r#" '\u{1}' "#); } #[test] fn literal_integer() { - assert_eq!(Literal::u8_suffixed(10).to_string(), "10u8"); - assert_eq!(Literal::u16_suffixed(10).to_string(), "10u16"); - assert_eq!(Literal::u32_suffixed(10).to_string(), "10u32"); - assert_eq!(Literal::u64_suffixed(10).to_string(), "10u64"); - assert_eq!(Literal::u128_suffixed(10).to_string(), "10u128"); - assert_eq!(Literal::usize_suffixed(10).to_string(), "10usize"); - - assert_eq!(Literal::i8_suffixed(10).to_string(), "10i8"); - assert_eq!(Literal::i16_suffixed(10).to_string(), "10i16"); - assert_eq!(Literal::i32_suffixed(10).to_string(), "10i32"); - assert_eq!(Literal::i64_suffixed(10).to_string(), "10i64"); - assert_eq!(Literal::i128_suffixed(10).to_string(), "10i128"); - assert_eq!(Literal::isize_suffixed(10).to_string(), "10isize"); - - assert_eq!(Literal::u8_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::u16_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::u32_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::u64_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::u128_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::usize_unsuffixed(10).to_string(), "10"); - - assert_eq!(Literal::i8_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::i16_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::i32_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::i64_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::i128_unsuffixed(10).to_string(), "10"); - assert_eq!(Literal::isize_unsuffixed(10).to_string(), "10"); + #[track_caller] + fn assert(literal: Literal, expected: &str) { + assert_eq!(literal.to_string(), expected); + } + + assert(Literal::u8_suffixed(10), "10u8"); + assert(Literal::u16_suffixed(10), "10u16"); + assert(Literal::u32_suffixed(10), "10u32"); + assert(Literal::u64_suffixed(10), "10u64"); + assert(Literal::u128_suffixed(10), "10u128"); + assert(Literal::usize_suffixed(10), "10usize"); + + assert(Literal::i8_suffixed(10), "10i8"); + assert(Literal::i16_suffixed(10), "10i16"); + assert(Literal::i32_suffixed(10), "10i32"); + assert(Literal::i64_suffixed(10), "10i64"); + assert(Literal::i128_suffixed(10), "10i128"); + assert(Literal::isize_suffixed(10), "10isize"); + + assert(Literal::u8_unsuffixed(10), "10"); + assert(Literal::u16_unsuffixed(10), "10"); + assert(Literal::u32_unsuffixed(10), "10"); + assert(Literal::u64_unsuffixed(10), "10"); + assert(Literal::u128_unsuffixed(10), "10"); + assert(Literal::usize_unsuffixed(10), "10"); + + assert(Literal::i8_unsuffixed(10), "10"); + assert(Literal::i16_unsuffixed(10), "10"); + assert(Literal::i32_unsuffixed(10), "10"); + assert(Literal::i64_unsuffixed(10), "10"); + assert(Literal::i128_unsuffixed(10), "10"); + assert(Literal::isize_unsuffixed(10), "10"); + + assert(Literal::i32_suffixed(-10), "-10i32"); + assert(Literal::i32_unsuffixed(-10), "-10"); } #[test] fn literal_float() { - assert_eq!(Literal::f32_suffixed(10.0).to_string(), "10f32"); - assert_eq!(Literal::f64_suffixed(10.0).to_string(), "10f64"); + #[track_caller] + fn assert(literal: Literal, expected: &str) { + assert_eq!(literal.to_string(), expected); + } + + assert(Literal::f32_suffixed(10.0), "10f32"); + assert(Literal::f32_suffixed(-10.0), "-10f32"); + assert(Literal::f64_suffixed(10.0), "10f64"); + assert(Literal::f64_suffixed(-10.0), "-10f64"); + + assert(Literal::f32_unsuffixed(10.0), "10.0"); + assert(Literal::f32_unsuffixed(-10.0), "-10.0"); + assert(Literal::f64_unsuffixed(10.0), "10.0"); + assert(Literal::f64_unsuffixed(-10.0), "-10.0"); - assert_eq!(Literal::f32_unsuffixed(10.0).to_string(), "10.0"); - assert_eq!(Literal::f64_unsuffixed(10.0).to_string(), "10.0"); + assert( + Literal::f64_unsuffixed(1e100), + "10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.0", + ); } #[test] @@ -248,9 +354,13 @@ fn literal_suffix() { assert_eq!(token_count("1._m"), 3); assert_eq!(token_count("\"\"s"), 1); assert_eq!(token_count("r\"\"r"), 1); + assert_eq!(token_count("r#\"\"#r"), 1); assert_eq!(token_count("b\"\"b"), 1); assert_eq!(token_count("br\"\"br"), 1); - assert_eq!(token_count("r#\"\"#r"), 1); + assert_eq!(token_count("br#\"\"#br"), 1); + assert_eq!(token_count("c\"\"c"), 1); + assert_eq!(token_count("cr\"\"cr"), 1); + assert_eq!(token_count("cr#\"\"#cr"), 1); assert_eq!(token_count("'c'c"), 1); assert_eq!(token_count("b'b'b"), 1); assert_eq!(token_count("0E"), 1); @@ -378,7 +488,7 @@ fn roundtrip() { roundtrip("'a"); roundtrip("'_"); roundtrip("'static"); - roundtrip("'\\u{10__FFFF}'"); + roundtrip(r"'\u{10__FFFF}'"); roundtrip("\"\\u{10_F0FF__}foo\\u{1_0_0_0__}\""); } @@ -401,6 +511,7 @@ fn fail() { fail("\"\\\r \""); // backslash carriage return fail("'aa'aa"); fail("br##\"\"#"); + fail("cr##\"\"#"); fail("\"\\\n\u{85}\r\""); } @@ -757,3 +868,39 @@ fn byte_order_mark() { let string = "foo\u{feff}"; string.parse::<TokenStream>().unwrap_err(); } + +#[cfg(span_locations)] +fn create_span() -> proc_macro2::Span { + let tts: TokenStream = "1".parse().unwrap(); + match tts.into_iter().next().unwrap() { + TokenTree::Literal(literal) => literal.span(), + _ => unreachable!(), + } +} + +#[cfg(span_locations)] +#[test] +fn test_invalidate_current_thread_spans() { + let actual = format!("{:#?}", create_span()); + assert_eq!(actual, "bytes(1..2)"); + let actual = format!("{:#?}", create_span()); + assert_eq!(actual, "bytes(3..4)"); + + proc_macro2::extra::invalidate_current_thread_spans(); + + let actual = format!("{:#?}", create_span()); + // Test that span offsets have been reset after the call + // to invalidate_current_thread_spans() + assert_eq!(actual, "bytes(1..2)"); +} + +#[cfg(span_locations)] +#[test] +#[should_panic(expected = "Invalid span with no related FileInfo!")] +fn test_use_span_after_invalidation() { + let span = create_span(); + + proc_macro2::extra::invalidate_current_thread_spans(); + + span.source_text(); +} |