diff options
Diffstat (limited to 'third_party/rust/core-foundation-sys/src/string_tokenizer.rs')
-rw-r--r-- | third_party/rust/core-foundation-sys/src/string_tokenizer.rs | 91 |
1 files changed, 91 insertions, 0 deletions
diff --git a/third_party/rust/core-foundation-sys/src/string_tokenizer.rs b/third_party/rust/core-foundation-sys/src/string_tokenizer.rs new file mode 100644 index 0000000000..b23d6a2d34 --- /dev/null +++ b/third_party/rust/core-foundation-sys/src/string_tokenizer.rs @@ -0,0 +1,91 @@ +// Copyright 2023 The Servo Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::os::raw::c_void; + +use crate::array::CFMutableArrayRef; +use crate::base::{CFAllocatorRef, CFIndex, CFOptionFlags, CFRange, CFTypeID, CFTypeRef}; +use crate::locale::CFLocaleRef; +use crate::string::CFStringRef; + +#[repr(C)] +pub struct __CFStringTokenizer(c_void); +pub type CFStringTokenizerRef = *mut __CFStringTokenizer; + +pub type CFStringTokenizerTokenType = CFOptionFlags; + +pub const kCFStringTokenizerTokenNone: CFStringTokenizerTokenType = 0; +pub const kCFStringTokenizerTokenNormal: CFStringTokenizerTokenType = 1 << 0; +pub const kCFStringTokenizerTokenHasSubTokensMask: CFStringTokenizerTokenType = 1 << 1; +pub const kCFStringTokenizerTokenHasDerivedSubTokensMask: CFStringTokenizerTokenType = 1 << 2; +pub const kCFStringTokenizerTokenHasHasNumbersMask: CFStringTokenizerTokenType = 1 << 3; +pub const kCFStringTokenizerTokenHasNonLettersMask: CFStringTokenizerTokenType = 1 << 4; +pub const kCFStringTokenizerTokenIsCJWordMask: CFStringTokenizerTokenType = 1 << 5; + +/* Tokenization Modifiers */ +pub const kCFStringTokenizerUnitWord: CFOptionFlags = 0; +pub const kCFStringTokenizerUnitSentence: CFOptionFlags = 1; +pub const kCFStringTokenizerUnitParagraph: CFOptionFlags = 2; +pub const kCFStringTokenizerUnitLineBreak: CFOptionFlags = 3; +pub const kCFStringTokenizerUnitWordBoundary: CFOptionFlags = 4; +pub const kCFStringTokenizerAttributeLatinTranscription: CFOptionFlags = 1 << 16; +pub const kCFStringTokenizerAttributeLanguage: CFOptionFlags = 1 << 17; + +extern "C" { + /* + * CFStringTokenizer.h + */ + + /* Creating a Tokenizer */ + pub fn CFStringTokenizerCreate( + alloc: CFAllocatorRef, + string: CFStringRef, + range: CFRange, + options: CFOptionFlags, + locale: CFLocaleRef, + ) -> CFStringTokenizerRef; + + /* Setting the String */ + pub fn CFStringTokenizerSetString( + tokenizer: CFStringTokenizerRef, + string: CFStringRef, + range: CFRange, + ); + + /* Changing the Location */ + pub fn CFStringTokenizerAdvanceToNextToken( + tokenizer: CFStringTokenizerRef, + ) -> CFStringTokenizerTokenType; + pub fn CFStringTokenizerGoToTokenAtIndex( + tokenizer: CFStringTokenizerRef, + index: CFIndex, + ) -> CFStringTokenizerTokenType; + + /* Getting Information About the Current Token */ + pub fn CFStringTokenizerCopyCurrentTokenAttribute( + tokenizer: CFStringTokenizerRef, + attribute: CFOptionFlags, + ) -> CFTypeRef; + pub fn CFStringTokenizerGetCurrentTokenRange(tokenizer: CFStringTokenizerRef) -> CFRange; + pub fn CFStringTokenizerGetCurrentSubTokens( + tokenizer: CFStringTokenizerRef, + ranges: *mut CFRange, + maxRangeLength: CFIndex, + derivedSubTokens: CFMutableArrayRef, + ) -> CFIndex; + + /* Identifying a Language */ + pub fn CFStringTokenizerCopyBestStringLanguage( + string: CFStringRef, + range: CFRange, + ) -> CFStringRef; + + /* Getting the CFStringTokenizer Type ID */ + pub fn CFStringTokenizerGetTypeID() -> CFTypeID; +} |