diff options
Diffstat (limited to 'third_party/rust/mapped_hyph/src')
| -rw-r--r-- | third_party/rust/mapped_hyph/src/bin/hyf_compile.rs | 44 | ||||
| -rw-r--r-- | third_party/rust/mapped_hyph/src/builder.rs | 509 | ||||
| -rw-r--r-- | third_party/rust/mapped_hyph/src/ffi.rs | 250 | ||||
| -rw-r--r-- | third_party/rust/mapped_hyph/src/lib.rs | 642 | ||||
| -rw-r--r-- | third_party/rust/mapped_hyph/src/main.rs | 67 |
5 files changed, 1512 insertions, 0 deletions
diff --git a/third_party/rust/mapped_hyph/src/bin/hyf_compile.rs b/third_party/rust/mapped_hyph/src/bin/hyf_compile.rs new file mode 100644 index 0000000000..257c747f54 --- /dev/null +++ b/third_party/rust/mapped_hyph/src/bin/hyf_compile.rs @@ -0,0 +1,44 @@ +// Copyright 2019-2020 Mozilla Foundation. See the COPYRIGHT +// file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +extern crate log; +extern crate mapped_hyph; + +use std::env; +use std::fs::File; + +struct Logger {} + +impl log::Log for Logger { + fn enabled(&self, _: &log::Metadata) -> bool { + true + } + + fn log(&self, record: &log::Record) { + eprintln!("{} - {}", record.level(), record.args()); + } + + fn flush(&self) {} +} + +static LOGGER: Logger = Logger {}; + +fn main() -> std::io::Result<()> { + unsafe { log::set_logger_racy(&LOGGER).unwrap() }; + + let args: Vec<String> = env::args().collect(); + if args.len() == 3 { + let in_file = File::open(&args[1])?; + let mut out_file = File::create(&args[2])?; + mapped_hyph::builder::compile(&in_file, &mut out_file, true)?; + } else { + println!("usage: hyf_compile <pattern-file> <output-file>"); + } + Ok(()) +} diff --git a/third_party/rust/mapped_hyph/src/builder.rs b/third_party/rust/mapped_hyph/src/builder.rs new file mode 100644 index 0000000000..e19a0087fd --- /dev/null +++ b/third_party/rust/mapped_hyph/src/builder.rs @@ -0,0 +1,509 @@ +// Copyright 2019-2020 Mozilla Foundation. See the COPYRIGHT +// file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +/// Functions to compile human-readable patterns into a mapped_hyph +/// flattened representation of the hyphenation state machine. + +use std::io::{Read,BufRead,BufReader,Write,Error,ErrorKind}; +use std::collections::HashMap; +use std::convert::TryInto; +use std::hash::{Hash,Hasher}; + +// Wrap a HashMap so that we can implement the Hash trait. +#[derive(PartialEq, Eq, Clone)] +struct TransitionMap (HashMap<u8,i32>); + +impl TransitionMap { + fn new() -> TransitionMap { + TransitionMap(HashMap::<u8,i32>::new()) + } +} + +impl Hash for TransitionMap { + fn hash<H: Hasher>(&self, state: &mut H) { + // We only look at the values here; that's likely to be enough + // for a reasonable hash. + let mut transitions: Vec<&i32> = self.0.values().collect(); + transitions.sort(); + for t in transitions { + t.hash(state); + } + } +} + +#[derive(PartialEq, Eq, Hash, Clone)] +struct State { + match_string: Option<Vec<u8>>, + #[allow(dead_code)] + repl_string: Option<Vec<u8>>, + #[allow(dead_code)] + repl_index: i32, + #[allow(dead_code)] + repl_cut: i32, + fallback_state: i32, + transitions: TransitionMap, +} + +impl State { + fn new() -> State { + State { + match_string: None, + repl_string: None, + repl_index: -1, + repl_cut: -1, + fallback_state: -1, + transitions: TransitionMap::new(), + } + } +} + +/// Structures returned by the read_dic_file() function; +/// array of these can then be passed to write_hyf_file() +/// to create the flattened output. +struct LevelBuilder { + states: Vec<State>, + str_to_state: HashMap<Vec<u8>,i32>, + encoding: Option<String>, + nohyphen: Option<String>, + lh_min: u8, + rh_min: u8, + clh_min: u8, + crh_min: u8, +} + +impl LevelBuilder { + fn new() -> LevelBuilder { + let mut result = LevelBuilder { + states: Vec::<State>::new(), + str_to_state: HashMap::<Vec<u8>,i32>::new(), + encoding: None, + nohyphen: None, + lh_min: 0, + rh_min: 0, + clh_min: 0, + crh_min: 0, + }; + // Initialize the builder with an empty start state. + result.str_to_state.insert(vec![], 0); + result.states.push(State::new()); + result + } + + fn find_state_number_for(&mut self, text: &[u8]) -> i32 { + let count = self.states.len() as i32; + let index = *self.str_to_state.entry(text.to_vec()).or_insert(count); + if index == count { + self.states.push(State::new()); + } + index + } + + fn add_pattern(&mut self, pattern: &str) { + let mut bytes = pattern.as_bytes(); + let mut text = Vec::<u8>::with_capacity(bytes.len()); + let mut digits = Vec::<u8>::with_capacity(bytes.len() + 1); + let mut repl_str = None; + let mut repl_index = 0; + let mut repl_cut = 0; + + // Check for replacement rule (non-standard hyphenation spelling change). + if let Some(slash) = bytes.iter().position(|x| *x == b'/') { + let parts = bytes.split_at(slash); + bytes = parts.0; + let mut it = parts.1[1 ..].split(|x| *x == b','); + if let Some(repl) = it.next() { + repl_str = Some(repl.to_vec()); + } + if let Some(num) = it.next() { + repl_index = std::str::from_utf8(num).unwrap().parse::<i32>().unwrap() - 1; + } + if let Some(num) = it.next() { + repl_cut = std::str::from_utf8(num).unwrap().parse::<i32>().unwrap(); + } + } + + // Separate the input pattern into parallel arrays of text (bytes) and digits. + let mut got_digit = false; + for byte in bytes { + if *byte <= b'9' && *byte >= b'0' { + if got_digit { + warn!("invalid pattern \"{}\": consecutive digits", pattern); + return; + } + digits.push(*byte); + got_digit = true; + } else { + text.push(*byte); + if got_digit { + got_digit = false; + } else { + digits.push(b'0'); + } + } + } + if !got_digit { + digits.push(b'0'); + } + + if repl_str.is_none() { + // Optimize away leading zeroes from the digits array. + while !digits.is_empty() && digits[0] == b'0' { + digits.remove(0); + } + } else { + // Convert repl_index and repl_cut from Unicode char to byte indexing. + let start = if text[0] == b'.' { 1 } else { 0 }; + if start == 1 { + if digits[0] != b'0' { + warn!("invalid pattern \"{}\": unexpected digit before start of word", pattern); + return; + } + digits.remove(0); + } + let word = std::str::from_utf8(&text[start..]).unwrap(); + let mut chars: Vec<_> = word.char_indices().collect(); + chars.push((word.len(), '.')); + repl_cut = chars[(repl_index + repl_cut) as usize].0 as i32 - chars[repl_index as usize].0 as i32; + repl_index = chars[repl_index as usize].0 as i32; + } + + // Create the new state, or add pattern into an existing state + // (which should not already have a match_string). + let mut state_num = self.find_state_number_for(&text); + let mut state = &mut self.states[state_num as usize]; + if state.match_string.is_some() { + warn!("duplicate pattern \"{}\" discarded", pattern); + return; + } + if !digits.is_empty() { + state.match_string = Some(digits); + } + if repl_str.is_some() { + state.repl_string = repl_str; + state.repl_index = repl_index; + state.repl_cut = repl_cut; + } + + // Set up prefix transitions, inserting additional states as needed. + while !text.is_empty() { + let last_state = state_num; + let ch = *text.last().unwrap(); + text.truncate(text.len() - 1); + state_num = self.find_state_number_for(&text); + if let Some(exists) = self.states[state_num as usize].transitions.0.insert(ch, last_state) { + assert_eq!(exists, last_state, "overwriting existing transition at pattern \"{}\"", pattern); + break; + } + } + } + + fn merge_duplicate_states(&mut self) { + // We loop here because when we eliminate a duplicate, and update the transitons + // that referenced it, we may thereby create new duplicates that another pass + // will find and compress further. + loop { + let orig_len = self.states.len(); + // Used to map State records to the (first) index at which they occur. + let mut state_to_index = HashMap::<&State,i32>::new(); + // Mapping of old->new state indexes, and whether each old state is + // a duplicate that should be dropped. + let mut mappings = Vec::<(i32,bool)>::with_capacity(orig_len); + let mut next_new_index: i32 = 0; + for index in 0 .. self.states.len() { + // Find existing index for this state, or allocate the next new index to it. + let new_index = *state_to_index.entry(&self.states[index]).or_insert(next_new_index); + // Record the mapping, and whether the state was a duplicate. + mappings.push((new_index, new_index != next_new_index)); + // If we used next_new_index for this state, increment it. + if new_index == next_new_index { + next_new_index += 1; + } + } + // If we didn't find any duplicates, next_new_index will have kept pace with + // index, so we know we're finished. + if next_new_index as usize == self.states.len() { + break; + } + // Iterate over all the states, either deleting them or updating indexes + // according to the mapping we created; then repeat the search. + for index in (0 .. self.states.len()).rev() { + if mappings[index].1 { + self.states.remove(index); + } else { + let state = &mut self.states[index]; + if state.fallback_state != -1 { + state.fallback_state = mappings[state.fallback_state as usize].0; + } + for t in state.transitions.0.iter_mut() { + *t.1 = mappings[*t.1 as usize].0; + } + } + } + } + } + + fn flatten(&self) -> Vec<u8> { + // Calculate total space needed for state data, and build the state_to_offset table. + let mut state_data_size = 0; + let mut state_to_offset = Vec::<usize>::with_capacity(self.states.len()); + for state in &self.states { + state_to_offset.push(state_data_size); + state_data_size += if state.repl_string.is_some() { 12 } else { 8 }; + state_data_size += state.transitions.0.len() * 4; + } + + // Helper to map a state index to its offset in the final data block. + let get_state_offset_for = |state_index: i32| -> u32 { + if state_index < 0 { + return super::INVALID_STATE_OFFSET; + } + state_to_offset[state_index as usize] as u32 + }; + + // Helper to map a byte string to its offset in the final data block, and + // store the bytes into string_data unless using an already-existing string. + let mut string_to_offset = HashMap::<Vec<u8>,usize>::new(); + let mut string_data = Vec::<u8>::new(); + let mut get_string_offset_for = |bytes: &Option<Vec<u8>>| -> u16 { + if bytes.is_none() { + return super::INVALID_STRING_OFFSET; + } + assert!(bytes.as_ref().unwrap().len() < 256); + let new_offset = string_data.len(); + let offset = *string_to_offset.entry(bytes.as_ref().unwrap().clone()).or_insert(new_offset); + if offset == new_offset { + string_data.push(bytes.as_ref().unwrap().len() as u8); + string_data.extend_from_slice(bytes.as_ref().unwrap().as_ref()); + } + offset.try_into().unwrap() + }; + + // Handle nohyphen string list if present, converting comma separators to NULs + // and trimming any surplus whitespace. + let mut nohyphen_string_offset: u16 = super::INVALID_STRING_OFFSET; + let mut nohyphen_count: u16 = 0; + if self.nohyphen.is_some() { + let nohyphen_strings: Vec<_> = self.nohyphen.as_ref().unwrap().split(',').map(|x| x.trim()).collect(); + nohyphen_count = nohyphen_strings.len().try_into().unwrap(); + nohyphen_string_offset = get_string_offset_for(&Some(nohyphen_strings.join("\0").as_bytes().to_vec())); + } + + let mut state_data = Vec::<u8>::with_capacity(state_data_size); + for state in &self.states { + state_data.extend(&get_state_offset_for(state.fallback_state).to_le_bytes()); + state_data.extend(&get_string_offset_for(&state.match_string).to_le_bytes()); + state_data.push(state.transitions.0.len() as u8); + // Determine whether to use an extended state record, and if so add the + // replacement string and index fields. + if state.repl_string.is_none() { + state_data.push(0); + } else { + state_data.push(1); + state_data.extend(&get_string_offset_for(&state.repl_string).to_le_bytes()); + state_data.push(state.repl_index as u8); + state_data.push(state.repl_cut as u8); + } + // Collect transitions into an array so we can sort them. + let mut transitions = vec![]; + for (key, value) in state.transitions.0.iter() { + transitions.push((*key, get_state_offset_for(*value))) + } + transitions.sort(); + for t in transitions { + // New state offset is stored as a 24-bit value, so we do this manually. + state_data.push((t.1 & 0xff) as u8); + state_data.push(((t.1 >> 8) & 0xff) as u8); + state_data.push(((t.1 >> 16) & 0xff) as u8); + state_data.push(t.0); + } + } + assert_eq!(state_data.len(), state_data_size); + + // Pad string data to a 4-byte boundary + while string_data.len() & 3 != 0 { + string_data.push(0); + } + + let total_size = super::LEVEL_HEADER_SIZE as usize + state_data_size + string_data.len(); + let mut result = Vec::<u8>::with_capacity(total_size); + + let state_data_base: u32 = super::LEVEL_HEADER_SIZE as u32; + let string_data_base: u32 = state_data_base + state_data_size as u32; + + result.extend(&state_data_base.to_le_bytes()); + result.extend(&string_data_base.to_le_bytes()); + result.extend(&nohyphen_string_offset.to_le_bytes()); + result.extend(&nohyphen_count.to_le_bytes()); + result.push(self.lh_min); + result.push(self.rh_min); + result.push(self.clh_min); + result.push(self.crh_min); + + result.extend(state_data.iter()); + result.extend(string_data.iter()); + + assert_eq!(result.len(), total_size); + + result + } +} + +/// Read a libhyphen-style pattern file and create the corresponding state +/// machine transitions, etc. +/// The returned Vec can be passed to write_hyf_file() to generate a flattened +/// representation of the state machine in mapped_hyph's binary format. +fn read_dic_file<T: Read>(dic_file: T, compress: bool) -> Result<Vec<LevelBuilder>, &'static str> { + let reader = BufReader::new(dic_file); + + let mut builders = Vec::<LevelBuilder>::new(); + builders.push(LevelBuilder::new()); + let mut builder = &mut builders[0]; + + for (index, line) in reader.lines().enumerate() { + let mut trimmed = line.unwrap().trim().to_string(); + // Strip comments. + if let Some(i) = trimmed.find('%') { + trimmed = trimmed[..i].trim().to_string(); + } + // Ignore empty lines. + if trimmed.is_empty() { + continue; + } + // Uppercase indicates keyword rather than pattern. + if trimmed.as_bytes()[0] >= b'A' && trimmed.as_bytes()[0] <= b'Z' { + // First line is encoding; we only support UTF-8. + if builder.encoding.is_none() { + if trimmed != "UTF-8" { + return Err("Only UTF-8 patterns are accepted!"); + }; + builder.encoding = Some(trimmed); + continue; + } + // Check for valid keyword-value pairs. + if trimmed.contains(' ') { + let parts: Vec<&str> = trimmed.split(' ').collect(); + if parts.len() != 2 { + warn!("unrecognized keyword/values: {}", trimmed); + continue; + } + let keyword = parts[0]; + let value = parts[1]; + match keyword { + "LEFTHYPHENMIN" => builder.lh_min = value.parse::<u8>().unwrap(), + "RIGHTHYPHENMIN" => builder.rh_min = value.parse::<u8>().unwrap(), + "COMPOUNDLEFTHYPHENMIN" => builder.clh_min = value.parse::<u8>().unwrap(), + "COMPOUNDRIGHTHYPHENMIN" => builder.crh_min = value.parse::<u8>().unwrap(), + "NOHYPHEN" => builder.nohyphen = Some(trimmed), + _ => warn!("unknown keyword: {}", trimmed), + } + continue; + } + // Start a new hyphenation level? + if trimmed == "NEXTLEVEL" { + builders.push(LevelBuilder::new()); + builder = builders.last_mut().unwrap(); + continue; + } + warn!("unknown keyword: {}", trimmed); + continue; + } + // Patterns should always be provided in lowercase; complain if not, and discard + // the bad pattern. + if trimmed != trimmed.to_lowercase() { + warn!("pattern \"{}\" not lowercased at line {}", trimmed, index); + continue; + } + builder.add_pattern(&trimmed); + } + + // Create default first (compound-word) level if only one level was provided. + // (Maybe this should be optional? Currently just copying libhyphen behavior.) + if builders.len() == 1 { + let (lh_min, rh_min, clh_min, crh_min) = + (builders[0].lh_min, builders[0].rh_min, builders[0].clh_min, builders[0].crh_min); + builders.insert(0, LevelBuilder::new()); + builder = builders.first_mut().unwrap(); + builder.add_pattern("1-1"); + builder.add_pattern("1'1"); + builder.add_pattern("1\u{2013}1"); // en-dash + builder.add_pattern("1\u{2019}1"); // curly apostrophe + builder.nohyphen = Some("',\u{2013},\u{2019},-".to_string()); + builder.lh_min = lh_min; + builder.rh_min = rh_min; + builder.clh_min = if clh_min > 0 { clh_min } else if lh_min > 0 { lh_min } else { 3 }; + builder.crh_min = if crh_min > 0 { crh_min } else if rh_min > 0 { rh_min } else { 3 }; + } + + // Put in fallback states in each builder. + for builder in &mut builders { + for (key, state_index) in builder.str_to_state.iter() { + if key.is_empty() { + continue; + } + let mut fallback_key = key.clone(); + while !fallback_key.is_empty() { + fallback_key.remove(0); + if builder.str_to_state.contains_key(&fallback_key) { + break; + } + } + builder.states[*state_index as usize].fallback_state = builder.str_to_state[&fallback_key]; + } + } + + if compress { + // Merge duplicate states to reduce size. + for builder in &mut builders { + builder.merge_duplicate_states(); + } + } + + Ok(builders) +} + +/// Write out the state machines representing a set of hyphenation rules +/// to the given output stream. +fn write_hyf_file<T: Write>(hyf_file: &mut T, levels: Vec<LevelBuilder>) -> std::io::Result<()> { + if levels.is_empty() { + return Err(Error::from(ErrorKind::InvalidData)); + } + let mut flattened = vec![]; + for level in levels { + flattened.push(level.flatten()); + } + // Write file header: magic number, count of levels. + hyf_file.write_all(&[b'H', b'y', b'f', b'0'])?; + let level_count: u32 = flattened.len() as u32; + hyf_file.write_all(&level_count.to_le_bytes())?; + // Write array of offsets to each level. First level will begin immediately + // after the array of offsets. + let mut offset: u32 = super::FILE_HEADER_SIZE as u32 + 4 * level_count; + for flat in &flattened { + hyf_file.write_all(&offset.to_le_bytes())?; + offset += flat.len() as u32; + } + // Write the flattened data for each level. + for flat in &flattened { + hyf_file.write_all(&flat)?; + } + Ok(()) +} + +/// The public API to the compilation process: reads `dic_file` and writes compiled tables +/// to `hyf_file`. The `compress` param determines whether extra processing to reduce the +/// size of the output is performed. +pub fn compile<T1: Read, T2: Write>(dic_file: T1, hyf_file: &mut T2, compress: bool) -> std::io::Result<()> { + match read_dic_file(dic_file, compress) { + Ok(dic) => write_hyf_file(hyf_file, dic), + Err(e) => { + warn!("parse error: {}", e); + return Err(Error::from(ErrorKind::InvalidData)) + } + } +} diff --git a/third_party/rust/mapped_hyph/src/ffi.rs b/third_party/rust/mapped_hyph/src/ffi.rs new file mode 100644 index 0000000000..1b546e2567 --- /dev/null +++ b/third_party/rust/mapped_hyph/src/ffi.rs @@ -0,0 +1,250 @@ +// Copyright 2019-2020 Mozilla Foundation. See the COPYRIGHT +// file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::slice; +use std::str; +use std::ffi::CStr; +use std::fs::File; +use std::io::Read; +use std::os::raw::c_char; +use std::str::Utf8Error; + +use memmap2::Mmap; + +use super::Hyphenator; + +/// Opaque type representing a hyphenation dictionary loaded from a file, +/// for use in FFI function signatures. +pub struct HyphDic; + +/// Opaque type representing a compiled dictionary in a memory buffer. +pub struct CompiledData; + +// Helper to convert word and hyphen buffer parameters from raw C pointer/length +// pairs to the Rust types expected by mapped_hyph. +unsafe fn params_from_c<'a>(word: *const c_char, word_len: u32, + hyphens: *mut u8, hyphens_len: u32) -> + (Result<&'a str, Utf8Error>, &'a mut [u8]) { + (str::from_utf8(slice::from_raw_parts(word as *const u8, word_len as usize)), + slice::from_raw_parts_mut(hyphens, hyphens_len as usize)) +} + +/// C-callable function to load a hyphenation dictionary from a file at `path`. +/// +/// Returns null on failure. +/// +/// This does not fully validate that the file contains usable hyphenation +/// data, it only opens the file (read-only) and mmap's it into memory, and +/// does some minimal sanity-checking that it *might* be valid. +/// +/// The returned `HyphDic` must be released with `mapped_hyph_free_dictionary`. +/// +/// # Safety +/// The given `path` must be a valid pointer to a NUL-terminated (C-style) +/// string. +#[no_mangle] +pub unsafe extern "C" fn mapped_hyph_load_dictionary(path: *const c_char) -> *const HyphDic { + let path_str = match CStr::from_ptr(path).to_str() { + Ok(str) => str, + Err(_) => return std::ptr::null(), + }; + let hyph = Box::new(match super::load_file(path_str) { + Some(dic) => dic, + _ => return std::ptr::null(), + }); + Box::into_raw(hyph) as *const HyphDic +} + +/// C-callable function to free a hyphenation dictionary +/// that was loaded by `mapped_hyph_load_dictionary`. +/// +/// # Safety +/// The `dic` parameter must be a `HyphDic` pointer obtained from +/// `mapped_hyph_load_dictionary`, and not previously freed. +#[no_mangle] +pub unsafe extern "C" fn mapped_hyph_free_dictionary(dic: *mut HyphDic) { + Box::from_raw(dic); +} + +/// C-callable function to find hyphenation values for a given `word`, +/// using a dictionary loaded via `mapped_hyph_load_dictionary`. +/// +/// The `word` must be UTF-8-encoded, and is `word_len` bytes (not characters) +/// long. +/// +/// Caller must supply the `hyphens` output buffer for results; its size is +/// given in `hyphens_len`. +/// It should be at least `word_len` elements long. +/// +/// Returns -1 if `word` is not valid UTF-8, or the output `hyphens` buffer is +/// too small. +/// Otherwise returns the number of potential hyphenation positions found. +/// +/// # Panics +/// This function may panic if the given dictionary is not valid. +/// +/// # Safety +/// The `dic` parameter must be a `HyphDic` pointer obtained from +/// `mapped_hyph_load_dictionary`. +/// +/// The `word` and `hyphens` parameter must be valid pointers to memory buffers +/// of at least the respective sizes `word_len` and `hyphens_len`. +#[no_mangle] +pub unsafe extern "C" fn mapped_hyph_find_hyphen_values_dic(dic: *const HyphDic, + word: *const c_char, word_len: u32, + hyphens: *mut u8, hyphens_len: u32) -> i32 { + if word_len > hyphens_len { + return -1; + } + let (word_str, hyphen_buf) = params_from_c(word, word_len, hyphens, hyphens_len); + if word_str.is_err() { + return -1; + } + Hyphenator::new(&*(dic as *const Mmap)) + .find_hyphen_values(word_str.unwrap(), hyphen_buf) as i32 +} + +/// C-callable function to find hyphenation values for a given `word`, +/// using a dictionary loaded and owned by the caller. +/// +/// The dictionary is supplied as a raw memory buffer `dic_buf` of size +/// `dic_len`. +/// +/// The `word` must be UTF-8-encoded, and is `word_len` bytes (not characters) +/// long. +/// +/// Caller must supply the `hyphens` output buffer for results; its size is +/// given in `hyphens_len`. +/// It should be at least `word_len` elements long. +/// +/// Returns -1 if `word` is not valid UTF-8, or the output `hyphens` buffer is +/// too small. +/// Otherwise returns the number of potential hyphenation positions found. +/// +/// # Panics +/// This function may panic if the given dictionary is not valid. +/// +/// # Safety +/// The `dic_buf` parameter must be a valid pointer to a memory block of size +/// at least `dic_len`. +/// +/// The `word` and `hyphens` parameter must be valid pointers to memory buffers +/// of at least the respective sizes `word_len` and `hyphens_len`. +#[no_mangle] +pub unsafe extern "C" fn mapped_hyph_find_hyphen_values_raw(dic_buf: *const u8, dic_len: u32, + word: *const c_char, word_len: u32, + hyphens: *mut u8, hyphens_len: u32) -> i32 { + if word_len > hyphens_len { + return -1; + } + let (word_str, hyphen_buf) = params_from_c(word, word_len, hyphens, hyphens_len); + if word_str.is_err() { + return -1; + } + Hyphenator::new(slice::from_raw_parts(dic_buf, dic_len as usize)) + .find_hyphen_values(word_str.unwrap(), hyphen_buf) as i32 +} + +/// C-callable function to check if a given memory buffer `dic_buf` of size +/// `dic_len` is potentially usable as a hyphenation dictionary. +/// +/// Returns `true` if the given memory buffer looks like it may be a valid +/// hyphenation dictionary, `false` if it is clearly not usable. +/// +/// # Safety +/// The `dic_buf` parameter must be a valid pointer to a memory block of size +/// at least `dic_len`. +#[no_mangle] +pub unsafe extern "C" fn mapped_hyph_is_valid_hyphenator(dic_buf: *const u8, dic_len: u32) -> bool { + if dic_buf.is_null() { + return false; + } + let dic = Hyphenator::new(slice::from_raw_parts(dic_buf, dic_len as usize)); + dic.is_valid_hyphenator() +} + +/// C-callable function to free a CompiledData object created by +/// a `mapped_hyph_compile_...` function (below). +/// +/// # Safety +/// The `data` parameter must be a `CompiledData` pointer obtained from +/// a `mapped_hyph_compile_...` function, and not previously freed. +#[no_mangle] +pub unsafe extern "C" fn mapped_hyph_free_compiled_data(data: *mut CompiledData) { + Box::from_raw(data); +} + +// Helper for the compilation functions (from either memory buffer or file path). +fn compile_and_wrap<T: Read>(input: T, compress: bool) -> *const CompiledData { + let mut compiled: Vec<u8> = vec![]; + if super::builder::compile(input, &mut compiled, compress).is_err() { + return std::ptr::null(); + } + compiled.shrink_to_fit(); + + // Create a persistent heap reference to the compiled data, and return a pointer to it. + Box::into_raw(Box::new(compiled)) as *const CompiledData +} + +/// C-callable function to compile hyphenation patterns from `pattern_buf` and return +/// the compiled data in a memory buffer, suitable to be stored somewhere or passed +/// to `mapped_hyph_find_hyphen_values_raw` to perform hyphenation. +/// +/// The returned `CompiledData` must be released with `mapped_hyph_free_compiled_data`. +/// +/// # Safety +/// The `pattern_buf` parameter must be a valid pointer to a memory block of size +/// at least `pattern_len`. +#[no_mangle] +pub unsafe extern "C" fn mapped_hyph_compile_buffer(pattern_buf: *const u8, pattern_len: u32, compress: bool) -> *const CompiledData { + compile_and_wrap(slice::from_raw_parts(pattern_buf, pattern_len as usize), compress) +} + +/// C-callable function to compile hyphenation patterns from a file to a memory buffer. +/// +/// The returned `CompiledData` must be released with `mapped_hyph_free_compiled_data`. +/// +/// # Safety +/// The given `path` must be a valid pointer to a NUL-terminated (C-style) string. +#[no_mangle] +pub unsafe extern "C" fn mapped_hyph_compile_file(path: *const c_char, compress: bool) -> *const CompiledData { + // Try to open the file at the given path, returning null on failure. + let path_str = match CStr::from_ptr(path).to_str() { + Ok(str) => str, + Err(_) => return std::ptr::null(), + }; + let in_file = match File::open(path_str) { + Ok(file) => file, + Err(_) => return std::ptr::null(), + }; + compile_and_wrap(&in_file, compress) +} + +/// Get the size of the compiled table buffer in a `CompiledData` object. +/// +/// # Safety +/// The `data` parameter must be a `CompiledData` pointer obtained from +/// a `mapped_hyph_compile_...` function, and not previously freed. +#[no_mangle] +pub unsafe extern "C" fn mapped_hyph_compiled_data_size(data: *const CompiledData) -> u32 { + (&*(data as *const Vec<u8>)).len() as u32 +} + +/// Get a pointer to the raw data held by a `CompiledData` object. +/// +/// # Safety +/// The `data` parameter must be a `CompiledData` pointer obtained from +/// a `mapped_hyph_compile_...` function, and not previously freed. +/// +/// The returned pointer only remains valid as long as the `CompiledData` has not +/// been released (by passing it to `mapped_hyph_free_compiled_data`). +#[no_mangle] +pub unsafe extern "C" fn mapped_hyph_compiled_data_ptr(data: *const CompiledData) -> *const u8 { + (&*(data as *const Vec<u8>)).as_ptr() +} diff --git a/third_party/rust/mapped_hyph/src/lib.rs b/third_party/rust/mapped_hyph/src/lib.rs new file mode 100644 index 0000000000..848c93d257 --- /dev/null +++ b/third_party/rust/mapped_hyph/src/lib.rs @@ -0,0 +1,642 @@ +// Copyright 2019 Mozilla Foundation. See the COPYRIGHT +// file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#[macro_use] +extern crate arrayref; +extern crate memmap2; +#[macro_use] +extern crate log; + +use std::slice; +use std::str; +use std::cmp::max; +use std::fs::File; +use std::mem; + +use memmap2::Mmap; + +// Make submodules available publicly. +pub mod builder; +pub mod ffi; + +// 4-byte identification expected at beginning of a compiled dictionary file. +// (This will be updated if an incompatible change to the format is made in +// some future revision.) +const MAGIC_NUMBER: [u8; 4] = [b'H', b'y', b'f', b'0']; + +const INVALID_STRING_OFFSET: u16 = 0xffff; +const INVALID_STATE_OFFSET: u32 = 0x00ff_ffff; + +const FILE_HEADER_SIZE: usize = 8; // 4-byte magic number, 4-byte count of levels +const LEVEL_HEADER_SIZE: usize = 16; + +// Transition actually holds a 24-bit new state offset and an 8-bit input byte +// to match. We will be interpreting byte ranges as Transition arrays (in the +// State::transitions() method below), so use repr(C) to ensure we have the +// memory layout we expect. +// Transition records do not depend on any specific alignment. +#[repr(C)] +#[derive(Debug,Copy,Clone)] +struct Transition(u8, u8, u8, u8); + +impl Transition { + fn new_state_offset(&self) -> usize { + // Read a 24-bit little-endian number from three bytes. + self.0 as usize + ((self.1 as usize) << 8) + ((self.2 as usize) << 16) + } + fn match_byte(&self) -> u8 { + self.3 + } +} + +// State is an area of the Level's data block that begins with a fixed header, +// followed by an array of transitions. The total size of each State's data +// depends on the number of transitions in the state. Only the basic header +// is defined by the struct here; the rest of the state is accessed via +// pointer magic. +// There are two versions of State, a basic version that supports only simple +// hyphenation (no associated spelling change), and an extended version that +// adds the replacement-string fields to support spelling changes at the +// hyphenation point. Check is_extended() to know which version is present. +// State records are NOT necessarily 4-byte aligned, so multi-byte fields +// should be read with care. +#[derive(Debug,Copy,Clone)] +#[repr(C)] +struct State { + fallback_state: [u8; 4], + match_string_offset: [u8; 2], + num_transitions: u8, + is_extended: u8, +} + +#[repr(C)] +struct StateExtended { + state: State, + repl_string_offset: [u8; 2], + repl_index: i8, + repl_cut: i8, +} + +impl State { + // Accessors for the various State header fields; see file format description. + fn fallback_state(&self) -> usize { + u32::from_le_bytes(self.fallback_state) as usize + } + fn match_string_offset(&self) -> usize { + u16::from_le_bytes(self.match_string_offset) as usize + } + fn num_transitions(&self) -> u8 { + self.num_transitions + } + fn is_extended(&self) -> bool { + self.is_extended != 0 + } + // Accessors that are only valid if is_extended() is true. + // These use `unsafe` to dereference a pointer to the relevant field; + // this is OK because Level::get_state always validates the total state size + // before returning a state reference, so these pointers will be valid for + // any extended state it returns. + #[allow(dead_code)] + fn as_extended(&self) -> &StateExtended { + debug_assert!(self.is_extended()); + unsafe { mem::transmute(self) } + } + #[allow(dead_code)] + fn repl_string_offset(&self) -> usize { + u16::from_le_bytes(self.as_extended().repl_string_offset) as usize + } + #[allow(dead_code)] + fn repl_index(&self) -> i8 { + self.as_extended().repl_index + } + #[allow(dead_code)] + fn repl_cut(&self) -> i8 { + self.as_extended().repl_cut + } + // Return the state's Transitions as a slice reference. + fn transitions(&self) -> &[Transition] { + let count = self.num_transitions() as usize; + if count == 0 { + return &[]; + } + let transition_offset = if self.is_extended() { mem::size_of::<StateExtended>() } else { mem::size_of::<State>() } as isize; + // We know the `offset` here will not look beyond the valid range of memory + // because Level::get_state() checks the state length (accounting for the + // number of transitions) before returning a State reference. + let trans_ptr = unsafe { (self as *const State as *const u8).offset(transition_offset) as *const Transition }; + // Again, because Level::get_state() already checked the state length, we know + // this slice address and count will be valid. + unsafe { slice::from_raw_parts(trans_ptr, count) } + } + // Look up the Transition for a given input byte, or None. + fn transition_for(&self, b: u8) -> Option<Transition> { + // The transitions array is sorted by match_byte() value, but there are + // usually very few entries; benchmarking showed that using binary_search_by + // here gave no benefit (possibly slightly slower). + self.transitions().iter().copied().find(|t| t.match_byte() == b) + } + // Just for debugging use... + #[allow(dead_code)] + fn deep_show(&self, prefix: &str, dic: &Level) { + if self.match_string_offset() != INVALID_STRING_OFFSET as usize { + let match_string = dic.string_at_offset(self.match_string_offset()); + println!("{}match: {}", prefix, str::from_utf8(match_string).unwrap()); + } + for t in self.transitions() { + println!("{}{} ->", prefix, t.match_byte() as char); + let next_prefix = format!("{} ", prefix); + dic.get_state(t.new_state_offset()).unwrap().deep_show(&next_prefix, &dic); + } + } +} + +// We count the presentation-form ligature characters U+FB00..FB06 as multiple +// chars for the purposes of lefthyphenmin/righthyphenmin. In UTF-8, all these +// ligature characters are 3-byte sequences beginning with <0xEF, 0xAC>; this +// helper returns the "decomposed length" of the ligature given its trailing +// byte. +fn lig_length(trail_byte: u8) -> usize { + // This is only called on valid UTF-8 where we already know trail_byte + // must be >= 0x80. + // Ligature lengths: ff fi fl ffi ffl long-st st + const LENGTHS: [u8; 7] = [ 2u8, 2u8, 2u8, 3u8, 3u8, 2u8, 2u8 ]; + if trail_byte > 0x86 { + return 1; + } + LENGTHS[trail_byte as usize - 0x80] as usize +} + +fn is_utf8_trail_byte(byte: u8) -> bool { + (byte & 0xC0) == 0x80 +} + +fn is_ascii_digit(byte: u8) -> bool { + byte <= b'9' && byte >= b'0' +} + +fn is_odd(byte: u8) -> bool { + (byte & 0x01) == 0x01 +} + +// A hyphenation Level has a header followed by State records and packed string +// data. The total size of the slice depends on the number and size of the +// States and Strings it contains. +// Note that the data of the Level may not have any specific alignment! +#[derive(Debug,Copy,Clone)] +struct Level<'a> { + data: &'a [u8], + // Header fields cached by the constructor for faster access: + state_data_base_: usize, + string_data_base_: usize, +} + +impl Level<'_> { + // Constructor that initializes our cache variables. + fn new(data: &[u8]) -> Level { + Level { + data, + state_data_base_: u32::from_le_bytes(*array_ref!(data, 0, 4)) as usize, + string_data_base_: u32::from_le_bytes(*array_ref!(data, 4, 4)) as usize, + } + } + + // Accessors for Level header fields; see file format description. + fn state_data_base(&self) -> usize { + self.state_data_base_ // cached by constructor + } + fn string_data_base(&self) -> usize { + self.string_data_base_ // cached by constructor + } + fn nohyphen_string_offset(&self) -> usize { + u16::from_le_bytes(*array_ref!(self.data, 8, 2)) as usize + } + #[allow(dead_code)] + fn nohyphen_count(&self) -> u16 { + u16::from_le_bytes(*array_ref!(self.data, 10, 2)) + } + fn lh_min(&self) -> usize { + max(1, self.data[12] as usize) + } + fn rh_min(&self) -> usize { + max(1, self.data[13] as usize) + } + fn clh_min(&self) -> usize { + max(1, self.data[14] as usize) + } + fn crh_min(&self) -> usize { + max(1, self.data[15] as usize) + } + fn word_boundary_mins(&self) -> (usize, usize, usize, usize) { + (self.lh_min(), self.rh_min(), self.clh_min(), self.crh_min()) + } + // Strings are represented as offsets from the Level's string_data_base. + // This returns a byte slice referencing the string at a given offset, + // or an empty slice if invalid. + fn string_at_offset(&self, offset: usize) -> &'_ [u8] { + if offset == INVALID_STRING_OFFSET as usize { + return &[]; + } + let string_base = self.string_data_base() as usize + offset; + // TODO: move this to the validation function. + debug_assert!(string_base < self.data.len()); + if string_base + 1 > self.data.len() { + return &[]; + } + let len = self.data[string_base] as usize; + // TODO: move this to the validation function. + debug_assert!(string_base + 1 + len <= self.data.len()); + if string_base + 1 + len > self.data.len() { + return &[]; + } + self.data.get(string_base + 1 .. string_base + 1 + len).unwrap() + } + // The nohyphen field actually contains multiple NUL-separated substrings; + // return them as a vector of individual byte slices. + fn nohyphen(&self) -> Vec<&[u8]> { + let string_offset = self.nohyphen_string_offset(); + let nohyph_str = self.string_at_offset(string_offset as usize); + if nohyph_str.is_empty() { + return vec![]; + } + nohyph_str.split(|&b| b == 0).collect() + } + // States are represented as an offset from the Level's state_data_base. + // This returns a reference to the State at a given offset, or None if invalid. + fn get_state(&self, offset: usize) -> Option<&State> { + if offset == INVALID_STATE_OFFSET as usize { + return None; + } + debug_assert_eq!(offset & 3, 0); + let state_base = self.state_data_base() + offset; + // TODO: move this to the validation function. + debug_assert!(state_base + mem::size_of::<State>() <= self.string_data_base()); + if state_base + mem::size_of::<State>() > self.string_data_base() { + return None; + } + let state_ptr = &self.data[state_base] as *const u8 as *const State; + // This is safe because we just checked against self.string_data_base() above. + let state = unsafe { state_ptr.as_ref().unwrap() }; + let length = if state.is_extended() { mem::size_of::<StateExtended>() } else { mem::size_of::<State>() } + + mem::size_of::<Transition>() * state.num_transitions() as usize; + // TODO: move this to the validation function. + debug_assert!(state_base + length <= self.string_data_base()); + if state_base + length > self.string_data_base() { + return None; + } + // This is safe because we checked the full state length against self.string_data_base(). + unsafe { state_ptr.as_ref() } + } + // Sets hyphenation values (odd = potential break, even = no break) in values[], + // and returns the change in the number of odd values present, so the caller can + // keep track of the total number of potential breaks in the word. + fn find_hyphen_values(&self, word: &str, values: &mut [u8], lh_min: usize, rh_min: usize) -> isize { + // Bail out immediately if the word is too short to hyphenate. + if word.len() < lh_min + rh_min { + return 0; + } + let start_state = self.get_state(0); + let mut st = start_state; + let mut hyph_count = 0; + for i in 0 .. word.len() + 2 { + // Loop over the word by bytes, with a virtual '.' added at each end + // to match word-boundary patterns. + let b = if i == 0 || i == word.len() + 1 { b'.' } else { word.as_bytes()[i - 1] }; + loop { + // Loop to repeatedly fall back if we don't find a matching transition. + // Note that this could infinite-loop if there is a state whose fallback + // points to itself (or a cycle of fallbacks), but this would represent + // a table compilation error. + // (A potential validation function could check for fallback cycles.) + if st.is_none() { + st = start_state; + break; + } + let state = st.unwrap(); + if let Some(tr) = state.transition_for(b) { + // Found a transition for the current byte. Look up the new state; + // if it has a match_string, merge its weights into `values`. + st = self.get_state(tr.new_state_offset()); + if let Some(state) = st { + let match_offset = state.match_string_offset(); + if match_offset != INVALID_STRING_OFFSET as usize { + if state.is_extended() { + debug_assert!(false, "extended hyphenation not supported by this function"); + } else { + let match_str = self.string_at_offset(match_offset); + let offset = i + 1 - match_str.len(); + assert!(offset + match_str.len() <= word.len() + 2); + for (j, ch) in match_str.iter().enumerate() { + let index = offset + j; + if index >= lh_min && index <= word.len() - rh_min { + // lh_min and rh_min are guaranteed to be >= 1, + // so this will not try to access outside values[]. + let old_value = values[index - 1]; + let value = ch - b'0'; + if value > old_value { + if is_odd(old_value) != is_odd(value) { + // Adjust hyph_count for the change we're making + hyph_count += if is_odd(value) { 1 } else { -1 }; + } + values[index - 1] = value; + } + } + } + } + } + } + // We have handled the current input byte; leave the fallback loop + // and get next input. + break; + } + // No transition for the current byte; go to fallback state and try again. + st = self.get_state(state.fallback_state()); + } + } + + // If the word was not purely ASCII, or if the word begins/ends with + // digits, the use of lh_min and rh_min above may not have correctly + // excluded enough positions, so we need to fix things up here. + let mut index = 0; + let mut count = 0; + let word_bytes = word.as_bytes(); + let mut clear_hyphen_at = |i| { if is_odd(values[i]) { hyph_count -= 1; } values[i] = 0; }; + // Handle lh_min. + while count < lh_min - 1 && index < word_bytes.len() { + let byte = word_bytes[index]; + clear_hyphen_at(index); + if byte < 0x80 { + index += 1; + if is_ascii_digit(byte) { + continue; // ASCII digits don't count + } + } else if byte == 0xEF && word_bytes[index + 1] == 0xAC { + // Unicode presentation-form ligature characters, which we count as + // multiple chars for the purpose of lh_min/rh_min, all begin with + // 0xEF, 0xAC in UTF-8. + count += lig_length(word_bytes[index + 2]); + clear_hyphen_at(index + 1); + clear_hyphen_at(index + 2); + index += 3; + continue; + } else { + index += 1; + while index < word_bytes.len() && is_utf8_trail_byte(word_bytes[index]) { + clear_hyphen_at(index); + index += 1; + } + } + count += 1; + } + + // Handle rh_min. + count = 0; + index = word.len(); + while count < rh_min && index > 0 { + index -= 1; + let byte = word_bytes[index]; + if index < word.len() - 1 { + clear_hyphen_at(index); + } + if byte < 0x80 { + // Only count if not an ASCII digit + if !is_ascii_digit(byte) { + count += 1; + } + continue; + } + if is_utf8_trail_byte(byte) { + continue; + } + if byte == 0xEF && word_bytes[index + 1] == 0xAC { + // Presentation-form ligatures count as multiple chars. + count += lig_length(word_bytes[index + 2]); + continue; + } + count += 1; + } + + hyph_count + } +} + +/// Hyphenation engine encapsulating a language-specific set of patterns (rules) +/// that identify possible break positions within a word. +pub struct Hyphenator<'a>(&'a [u8]); + +impl Hyphenator<'_> { + /// Return a Hyphenator that wraps the given buffer. + /// This does *not* check that the given buffer is in fact a valid hyphenation table. + /// Use `is_valid_hyphenator()` to determine whether it is usable. + /// (Calling hyphenation methods on a Hyphenator that wraps arbitrary, + /// unvalidated data is not unsafe, but may panic.) + pub fn new(buffer: &[u8]) -> Hyphenator { + Hyphenator(buffer) + } + + // Internal implementation details + fn magic_number(&self) -> &[u8] { + &self.0[0 .. 4] + } + fn num_levels(&self) -> usize { + u32::from_le_bytes(*array_ref!(self.0, 4, 4)) as usize + } + fn level(&self, i: usize) -> Level { + let offset = u32::from_le_bytes(*array_ref!(self.0, FILE_HEADER_SIZE + 4 * i, 4)) as usize; + let limit = if i == self.num_levels() - 1 { + self.0.len() + } else { + u32::from_le_bytes(*array_ref!(self.0, FILE_HEADER_SIZE + 4 * i + 4, 4)) as usize + }; + debug_assert!(offset + LEVEL_HEADER_SIZE <= limit && limit <= self.0.len()); + debug_assert_eq!(offset & 3, 0); + debug_assert_eq!(limit & 3, 0); + Level::new(&self.0[offset .. limit]) + } + + /// Identify acceptable hyphenation positions in the given `word`. + /// + /// The caller-supplied `values` must be at least as long as the `word`. + /// + /// On return, any elements with an odd value indicate positions in the word + /// after which a hyphen could be inserted. + /// + /// Returns the number of possible hyphenation positions that were found. + /// + /// # Panics + /// If the given `values` slice is too small to hold the results. + /// + /// If the block of memory represented by `self.0` is not in fact a valid + /// hyphenation dictionary, this function may panic with an overflow or + /// array bounds violation. + pub fn find_hyphen_values(&self, word: &str, values: &mut [u8]) -> isize { + assert!(values.len() >= word.len()); + values.iter_mut().for_each(|x| *x = 0); + let top_level = self.level(0); + let (lh_min, rh_min, clh_min, crh_min) = top_level.word_boundary_mins(); + if word.len() < lh_min + rh_min { + return 0; + } + let mut hyph_count = top_level.find_hyphen_values(word, values, lh_min, rh_min); + let compound = hyph_count > 0; + // Subsequent levels are applied to fragments between potential breaks + // already found: + for l in 1 .. self.num_levels() { + let level = self.level(l); + if hyph_count > 0 { + let mut begin = 0; + let mut lh = lh_min; + // lh_min and rh_min are both guaranteed to be greater than zero, + // so this loop will not reach fully to the end of the word. + for i in lh_min - 1 .. word.len() - rh_min { + if is_odd(values[i]) { + if i > begin { + // We've found a component of a compound; + // clear the corresponding values and apply the new level. + // (These values must be even, so hyph_count is unchanged.) + values[begin .. i].iter_mut().for_each(|x| { + *x = 0; + }); + hyph_count += level.find_hyphen_values(&word[begin ..= i], + &mut values[begin ..= i], + lh, crh_min); + } + begin = i + 1; + lh = clh_min; + } + } + if begin == 0 { + // No compound-word breaks were found, just apply level to the whole word. + hyph_count += level.find_hyphen_values(word, values, lh_min, rh_min); + } else if begin < word.len() { + // Handle trailing component of compound. + hyph_count += level.find_hyphen_values(&word[begin .. word.len()], + &mut values[begin .. word.len()], + clh_min, rh_min); + } + } else { + hyph_count += level.find_hyphen_values(word, values, lh_min, rh_min); + } + } + + // Only need to check nohyphen strings if top-level (compound) breaks were found. + if compound && hyph_count > 0 { + let nohyph = top_level.nohyphen(); + if !nohyph.is_empty() { + for i in lh_min ..= word.len() - rh_min { + if is_odd(values[i - 1]) { + for nh in &nohyph { + if i + nh.len() <= word.len() && *nh == &word.as_bytes()[i .. i + nh.len()] { + values[i - 1] = 0; + hyph_count -= 1; + break; + } + if nh.len() <= i && *nh == &word.as_bytes()[i - nh.len() .. i] { + values[i - 1] = 0; + hyph_count -= 1; + break; + } + } + } + } + } + } + + hyph_count + } + + /// Generate the hyphenated form of a `word` by inserting the given `hyphen_char` + /// at each valid break position. + /// + /// # Panics + /// If the block of memory represented by `self` is not in fact a valid + /// hyphenation dictionary, this function may panic with an overflow or + /// array bounds violation. + /// + /// Also panics if the length of the hyphenated word would overflow `usize`. + pub fn hyphenate_word(&self, word: &str, hyphchar: char) -> String { + let mut values = vec![0u8; word.len()]; + let hyph_count = self.find_hyphen_values(word, &mut values); + if hyph_count <= 0 { + return word.to_string(); + } + // We know how long the result will be, so we can preallocate here. + let result_len = word.len() + hyph_count as usize * hyphchar.len_utf8(); + let mut result = String::with_capacity(result_len); + let mut n = 0; + for ch in word.char_indices() { + if ch.0 > 0 && is_odd(values[ch.0 - 1]) { + result.push(hyphchar); + n += 1; + } + result.push(ch.1); + } + debug_assert_eq!(n, hyph_count); + debug_assert_eq!(result_len, result.len()); + result + } + + /// Check if the block of memory looks like it could be a valid hyphenation + /// table. + pub fn is_valid_hyphenator(&self) -> bool { + // Size must be at least 4 bytes for magic_number + 4 bytes num_levels; + // smaller than this cannot be safely inspected. + if self.0.len() < FILE_HEADER_SIZE { + return false; + } + if self.magic_number() != MAGIC_NUMBER { + return false; + } + // For each level, there's a 4-byte offset in the header, and the level + // has its own 16-byte header, so we can check a minimum size again here. + let num_levels = self.num_levels(); + if self.0.len() < FILE_HEADER_SIZE + LEVEL_HEADER_SIZE * num_levels { + return false; + } + // Check that state_data_base and string_data_base for each hyphenation + // level are within range. + for l in 0 .. num_levels { + let level = self.level(l); + if level.state_data_base() < LEVEL_HEADER_SIZE || + level.state_data_base() > level.string_data_base() || + level.string_data_base() > level.data.len() { + return false; + } + // TODO: consider doing more extensive validation of states and + // strings within the level? + } + // It's still possible the dic is internally broken, but at least it's + // worth trying to use it! + true + } +} + +/// Load the compiled hyphenation file at `dic_path`, if present. +/// +/// Returns `None` if the specified file cannot be opened or mapped, +/// otherwise returns a `memmap2::Mmap` mapping the file. +/// +/// # Safety +/// +/// This is unsafe for the same reason `Mmap::map()` is unsafe: +/// mapped_hyph does not guarantee safety if the mapped file is modified +/// (e.g. by another process) while we're using it. +/// +/// This verifies that the file looks superficially like it may be a +/// compiled hyphenation table, but does *not* fully check the validity +/// of the file contents! Calling hyphenation functions with the returned +/// data is not unsafe, but may panic if the data is invalid. +pub unsafe fn load_file(dic_path: &str) -> Option<Mmap> { + let file = File::open(dic_path).ok()?; + let dic = Mmap::map(&file).ok()?; + let hyph = Hyphenator(&*dic); + if hyph.is_valid_hyphenator() { + return Some(dic); + } + None +} diff --git a/third_party/rust/mapped_hyph/src/main.rs b/third_party/rust/mapped_hyph/src/main.rs new file mode 100644 index 0000000000..acc24babee --- /dev/null +++ b/third_party/rust/mapped_hyph/src/main.rs @@ -0,0 +1,67 @@ +// Copyright 2019 Mozilla Foundation. See the COPYRIGHT +// file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +extern crate mapped_hyph; + +use mapped_hyph::Hyphenator; + +fn main() { + let dic_path = "hyph_en_US.hyf"; + + let dic = match unsafe { mapped_hyph::load_file(dic_path) } { + Some(dic) => dic, + _ => panic!("failed to load dictionary {}", dic_path), + }; + let hyph = Hyphenator::new(&*dic); + + println!("{}", hyph.hyphenate_word("haha", '-')); + println!("{}", hyph.hyphenate_word("hahaha", '-')); + println!("{}", hyph.hyphenate_word("photo", '-')); + println!("{}", hyph.hyphenate_word("photograph", '-')); + println!("{}", hyph.hyphenate_word("photographer", '-')); + println!("{}", hyph.hyphenate_word("photographic", '-')); + println!("{}", hyph.hyphenate_word("photographical", '-')); + println!("{}", hyph.hyphenate_word("photographically", '-')); + println!("{}", hyph.hyphenate_word("supercalifragilisticexpialidocious", '-')); + println!("{}", hyph.hyphenate_word("o'dwyer", '=')); + println!("{}", hyph.hyphenate_word("o'callahan", '=')); + println!("{}", hyph.hyphenate_word("o’dwyer", '=')); + println!("{}", hyph.hyphenate_word("o’callahan", '=')); + println!("{}", hyph.hyphenate_word("petti-fogging", '=')); + println!("{}", hyph.hyphenate_word("e-mailing", '=')); + println!("{}", hyph.hyphenate_word("-x-mailing", '=')); + println!("{}", hyph.hyphenate_word("-strikeout-", '=')); + + let dic2 = match unsafe { mapped_hyph::load_file("tests/compound.hyf") } { + Some(dic) => dic, + _ => panic!("failed to load dictionary {}", "tests/compound.hyf"), + }; + + let h2 = Hyphenator::new(&*dic2); + println!("{}", h2.hyphenate_word("motorcycle", '=')); + + let dic3 = match unsafe { mapped_hyph::load_file("tests/rhmin.hyf") } { + Some(dic) => dic, + _ => panic!("failed to load dictionary {}", dic_path), + }; + let h3 = Hyphenator::new(&*dic3); + println!("{}", h3.hyphenate_word("övéit", '=')); + println!("{}", h3.hyphenate_word("అంగడిధర", '=')); + + let dic4 = match unsafe { mapped_hyph::load_file("tests/num.hyf") } { + Some(dic) => dic, + _ => panic!("failed to load dictionary {}", "tests/num.hyf"), + }; + let h4 = Hyphenator::new(&*dic4); + + println!("{}", h4.hyphenate_word("123foobar123", '=')); + println!("{}", h4.hyphenate_word("123foobarfoobar", '=')); + println!("{}", h4.hyphenate_word("foobarfoobar123", '=')); + println!("{}", h4.hyphenate_word("123foobarfoobar123", '=')); +} |