"""Utility functions common to the C and C++ domains.""" from __future__ import annotations import re from copy import deepcopy from typing import TYPE_CHECKING, Any, Callable from docutils import nodes from sphinx import addnodes from sphinx.util import logging if TYPE_CHECKING: from docutils.nodes import TextElement from sphinx.config import Config logger = logging.getLogger(__name__) StringifyTransform = Callable[[Any], str] _whitespace_re = re.compile(r'\s+') anon_identifier_re = re.compile(r'(@[a-zA-Z0-9_])[a-zA-Z0-9_]*\b') identifier_re = re.compile(r''' ( # This 'extends' _anon_identifier_re with the ordinary identifiers, # make sure they are in sync. (~?\b[a-zA-Z_]) # ordinary identifiers | (@[a-zA-Z0-9_]) # our extension for names of anonymous entities ) [a-zA-Z0-9_]*\b ''', flags=re.VERBOSE) integer_literal_re = re.compile(r'[1-9][0-9]*(\'[0-9]+)*') octal_literal_re = re.compile(r'0[0-7]*(\'[0-7]+)*') hex_literal_re = re.compile(r'0[xX][0-9a-fA-F]+(\'[0-9a-fA-F]+)*') binary_literal_re = re.compile(r'0[bB][01]+(\'[01]+)*') integers_literal_suffix_re = re.compile(r''' # unsigned and/or (long) long, in any order, but at least one of them ( ([uU] ([lL] | (ll) | (LL))?) | (([lL] | (ll) | (LL)) [uU]?) )\b # the ending word boundary is important for distinguishing # between suffixes and UDLs in C++ ''', flags=re.VERBOSE) float_literal_re = re.compile(r''' [+-]?( # decimal ([0-9]+(\'[0-9]+)*[eE][+-]?[0-9]+(\'[0-9]+)*) | (([0-9]+(\'[0-9]+)*)?\.[0-9]+(\'[0-9]+)*([eE][+-]?[0-9]+(\'[0-9]+)*)?) | ([0-9]+(\'[0-9]+)*\.([eE][+-]?[0-9]+(\'[0-9]+)*)?) # hex | (0[xX][0-9a-fA-F]+(\'[0-9a-fA-F]+)*[pP][+-]?[0-9a-fA-F]+(\'[0-9a-fA-F]+)*) | (0[xX]([0-9a-fA-F]+(\'[0-9a-fA-F]+)*)?\. [0-9a-fA-F]+(\'[0-9a-fA-F]+)*([pP][+-]?[0-9a-fA-F]+(\'[0-9a-fA-F]+)*)?) | (0[xX][0-9a-fA-F]+(\'[0-9a-fA-F]+)*\.([pP][+-]?[0-9a-fA-F]+(\'[0-9a-fA-F]+)*)?) ) ''', flags=re.VERBOSE) float_literal_suffix_re = re.compile(r'[fFlL]\b') # the ending word boundary is important for distinguishing between suffixes and UDLs in C++ char_literal_re = re.compile(r''' ((?:u8)|u|U|L)? '( (?:[^\\']) | (\\( (?:['"?\\abfnrtv]) | (?:[0-7]{1,3}) | (?:x[0-9a-fA-F]{2}) | (?:u[0-9a-fA-F]{4}) | (?:U[0-9a-fA-F]{8}) )) )' ''', flags=re.VERBOSE) def verify_description_mode(mode: str) -> None: if mode not in ('lastIsName', 'noneIsName', 'markType', 'markName', 'param', 'udl'): raise Exception("Description mode '%s' is invalid." % mode) class NoOldIdError(Exception): # Used to avoid implementing unneeded id generation for old id schemes. pass class ASTBaseBase: def __eq__(self, other: Any) -> bool: if type(self) is not type(other): return False try: for key, value in self.__dict__.items(): if value != getattr(other, key): return False except AttributeError: return False return True # Defining __hash__ = None is not strictly needed when __eq__ is defined. __hash__ = None # type: ignore[assignment] def clone(self) -> Any: return deepcopy(self) def _stringify(self, transform: StringifyTransform) -> str: raise NotImplementedError(repr(self)) def __str__(self) -> str: return self._stringify(lambda ast: str(ast)) def get_display_string(self) -> str: return self._stringify(lambda ast: ast.get_display_string()) def __repr__(self) -> str: return '<%s>' % self.__class__.__name__ ################################################################################ # Attributes ################################################################################ class ASTAttribute(ASTBaseBase): def describe_signature(self, signode: TextElement) -> None: raise NotImplementedError(repr(self)) class ASTCPPAttribute(ASTAttribute): def __init__(self, arg: str) -> None: self.arg = arg def _stringify(self, transform: StringifyTransform) -> str: return "[[" + self.arg + "]]" def describe_signature(self, signode: TextElement) -> None: signode.append(addnodes.desc_sig_punctuation('[[', '[[')) signode.append(nodes.Text(self.arg)) signode.append(addnodes.desc_sig_punctuation(']]', ']]')) class ASTGnuAttribute(ASTBaseBase): def __init__(self, name: str, args: ASTBaseParenExprList | None) -> None: self.name = name self.args = args def _stringify(self, transform: StringifyTransform) -> str: res = [self.name] if self.args: res.append(transform(self.args)) return ''.join(res) class ASTGnuAttributeList(ASTAttribute): def __init__(self, attrs: list[ASTGnuAttribute]) -> None: self.attrs = attrs def _stringify(self, transform: StringifyTransform) -> str: res = ['__attribute__(('] first = True for attr in self.attrs: if not first: res.append(', ') first = False res.append(transform(attr)) res.append('))') return ''.join(res) def describe_signature(self, signode: TextElement) -> None: txt = str(self) signode.append(nodes.Text(txt)) class ASTIdAttribute(ASTAttribute): """For simple attributes defined by the user.""" def __init__(self, id: str) -> None: self.id = id def _stringify(self, transform: StringifyTransform) -> str: return self.id def describe_signature(self, signode: TextElement) -> None: signode.append(nodes.Text(self.id)) class ASTParenAttribute(ASTAttribute): """For paren attributes defined by the user.""" def __init__(self, id: str, arg: str) -> None: self.id = id self.arg = arg def _stringify(self, transform: StringifyTransform) -> str: return self.id + '(' + self.arg + ')' def describe_signature(self, signode: TextElement) -> None: txt = str(self) signode.append(nodes.Text(txt)) class ASTAttributeList(ASTBaseBase): def __init__(self, attrs: list[ASTAttribute]) -> None: self.attrs = attrs def __len__(self) -> int: return len(self.attrs) def __add__(self, other: ASTAttributeList) -> ASTAttributeList: return ASTAttributeList(self.attrs + other.attrs) def _stringify(self, transform: StringifyTransform) -> str: return ' '.join(transform(attr) for attr in self.attrs) def describe_signature(self, signode: TextElement) -> None: if len(self.attrs) == 0: return self.attrs[0].describe_signature(signode) if len(self.attrs) == 1: return for attr in self.attrs[1:]: signode.append(addnodes.desc_sig_space()) attr.describe_signature(signode) ################################################################################ class ASTBaseParenExprList(ASTBaseBase): pass ################################################################################ class UnsupportedMultiCharacterCharLiteral(Exception): pass class DefinitionError(Exception): pass class BaseParser: def __init__(self, definition: str, *, location: nodes.Node | tuple[str, int] | str, config: Config) -> None: self.definition = definition.strip() self.location = location # for warnings self.config = config self.pos = 0 self.end = len(self.definition) self.last_match: re.Match[str] | None = None self._previous_state: tuple[int, re.Match[str] | None] = (0, None) self.otherErrors: list[DefinitionError] = [] # in our tests the following is set to False to capture bad parsing self.allowFallbackExpressionParsing = True def _make_multi_error(self, errors: list[Any], header: str) -> DefinitionError: if len(errors) == 1: if len(header) > 0: return DefinitionError(header + '\n' + str(errors[0][0])) else: return DefinitionError(str(errors[0][0])) result = [header, '\n'] for e in errors: if len(e[1]) > 0: indent = ' ' result.append(e[1]) result.append(':\n') for line in str(e[0]).split('\n'): if len(line) == 0: continue result.append(indent) result.append(line) result.append('\n') else: result.append(str(e[0])) return DefinitionError(''.join(result)) @property def language(self) -> str: raise NotImplementedError def status(self, msg: str) -> None: # for debugging indicator = '-' * self.pos + '^' logger.debug(f"{msg}\n{self.definition}\n{indicator}") # NoQA: G004 def fail(self, msg: str) -> None: errors = [] indicator = '-' * self.pos + '^' exMain = DefinitionError( 'Invalid %s declaration: %s [error at %d]\n %s\n %s' % (self.language, msg, self.pos, self.definition, indicator)) errors.append((exMain, "Main error")) for err in self.otherErrors: errors.append((err, "Potential other error")) self.otherErrors = [] raise self._make_multi_error(errors, '') def warn(self, msg: str) -> None: logger.warning(msg, location=self.location) def match(self, regex: re.Pattern[str]) -> bool: match = regex.match(self.definition, self.pos) if match is not None: self._previous_state = (self.pos, self.last_match) self.pos = match.end() self.last_match = match return True return False def skip_string(self, string: str) -> bool: strlen = len(string) if self.definition[self.pos:self.pos + strlen] == string: self.pos += strlen return True return False def skip_word(self, word: str) -> bool: return self.match(re.compile(r'\b%s\b' % re.escape(word))) def skip_ws(self) -> bool: return self.match(_whitespace_re) def skip_word_and_ws(self, word: str) -> bool: if self.skip_word(word): self.skip_ws() return True return False def skip_string_and_ws(self, string: str) -> bool: if self.skip_string(string): self.skip_ws() return True return False @property def eof(self) -> bool: return self.pos >= self.end @property def current_char(self) -> str: try: return self.definition[self.pos] except IndexError: return 'EOF' @property def matched_text(self) -> str: if self.last_match is not None: return self.last_match.group() return '' def read_rest(self) -> str: rv = self.definition[self.pos:] self.pos = self.end return rv def assert_end(self, *, allowSemicolon: bool = False) -> None: self.skip_ws() if allowSemicolon: if not self.eof and self.definition[self.pos:] != ';': self.fail('Expected end of definition or ;.') else: if not self.eof: self.fail('Expected end of definition.') ################################################################################ @property def id_attributes(self): raise NotImplementedError @property def paren_attributes(self): raise NotImplementedError def _parse_balanced_token_seq(self, end: list[str]) -> str: # TODO: add handling of string literals and similar brackets = {'(': ')', '[': ']', '{': '}'} startPos = self.pos symbols: list[str] = [] while not self.eof: if len(symbols) == 0 and self.current_char in end: break if self.current_char in brackets: symbols.append(brackets[self.current_char]) elif len(symbols) > 0 and self.current_char == symbols[-1]: symbols.pop() elif self.current_char in ")]}": self.fail("Unexpected '%s' in balanced-token-seq." % self.current_char) self.pos += 1 if self.eof: self.fail("Could not find end of balanced-token-seq starting at %d." % startPos) return self.definition[startPos:self.pos] def _parse_attribute(self) -> ASTAttribute | None: self.skip_ws() # try C++11 style startPos = self.pos if self.skip_string_and_ws('['): if not self.skip_string('['): self.pos = startPos else: # TODO: actually implement the correct grammar arg = self._parse_balanced_token_seq(end=[']']) if not self.skip_string_and_ws(']'): self.fail("Expected ']' in end of attribute.") if not self.skip_string_and_ws(']'): self.fail("Expected ']' in end of attribute after [[...]") return ASTCPPAttribute(arg) # try GNU style if self.skip_word_and_ws('__attribute__'): if not self.skip_string_and_ws('('): self.fail("Expected '(' after '__attribute__'.") if not self.skip_string_and_ws('('): self.fail("Expected '(' after '__attribute__('.") attrs = [] while 1: if self.match(identifier_re): name = self.matched_text exprs = self._parse_paren_expression_list() attrs.append(ASTGnuAttribute(name, exprs)) if self.skip_string_and_ws(','): continue if self.skip_string_and_ws(')'): break self.fail("Expected identifier, ')', or ',' in __attribute__.") if not self.skip_string_and_ws(')'): self.fail("Expected ')' after '__attribute__((...)'") return ASTGnuAttributeList(attrs) # try the simple id attributes defined by the user for id in self.id_attributes: if self.skip_word_and_ws(id): return ASTIdAttribute(id) # try the paren attributes defined by the user for id in self.paren_attributes: if not self.skip_string_and_ws(id): continue if not self.skip_string('('): self.fail("Expected '(' after user-defined paren-attribute.") arg = self._parse_balanced_token_seq(end=[')']) if not self.skip_string(')'): self.fail("Expected ')' to end user-defined paren-attribute.") return ASTParenAttribute(id, arg) return None def _parse_attribute_list(self) -> ASTAttributeList: res = [] while True: attr = self._parse_attribute() if attr is None: break res.append(attr) return ASTAttributeList(res) def _parse_paren_expression_list(self) -> ASTBaseParenExprList | None: raise NotImplementedError