10 files changed, 589 insertions, 0 deletions
diff --git a/tools/glsl_preproc/macros.py b/tools/glsl_preproc/macros.py
new file mode 100644
index 0000000..2ba7e21
--- /dev/null
+++ b/tools/glsl_preproc/macros.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+
+import re
+
+from variables import Var
+from templates import *
+from statement import *
+
+PATTERN_PRAGMA  = re.compile(flags=re.VERBOSE, pattern=r'''
+\s*\#\s*pragma\s+               # '#pragma'
+(?P<pragma>(?:                  # pragma name
+    GLSL[PHF]?
+))\s*
+(?P<rest>.*)$                   # rest of line (pragma body)
+''')
+
+# Represents a single #pragma macro
+class Macro(object):
+    PRAGMAS = {
+        'GLSL':  'SH_BUF_BODY',
+        'GLSLP': 'SH_BUF_PRELUDE',
+        'GLSLH': 'SH_BUF_HEADER',
+        'GLSLF': 'SH_BUF_FOOTER',
+    }
+
+    def __init__(self, linenr=0, type='GLSL'):
+        self.linenr = linenr
+        self.buf    = Macro.PRAGMAS[type]
+        self.name   = '_glsl_' + str(linenr)
+        self.body   = []    # list of statements
+        self.last   = None  # previous GLSLBlock (if unterminated)
+        self.vars   = VarSet()
+
+    def needs_single_line(self):
+        if not self.body:
+            return False
+        prev = self.body[-1]
+        return isinstance(prev, BlockStart) and not prev.multiline
+
+    def push_line(self, line):
+        self.vars.merge(line.vars)
+
+        if isinstance(line, GLSLLine):
+            if self.last:
+                self.last.append(line)
+            elif self.needs_single_line():
+                self.body.append(GLSLBlock(line))
+            else:
+                # start new GLSL block
+                self.last = GLSLBlock(line)
+                self.body.append(self.last)
+        else:
+            self.body.append(line)
+            self.last = None
+
+    def render_struct(self):
+        return STRUCT_TEMPLATE.render(macro=self)
+
+    def render_call(self):
+        return CALL_TEMPLATE.render(macro=self)
+
+    def render_fun(self):
+        return FUNCTION_TEMPLATE.render(macro=self, Var=Var)
+
+    # yields output lines
+    @staticmethod
+    def process_file(lines, strip=False):
+        macro = None
+        macros = []
+
+        for linenr, line_orig in enumerate(lines, start=1):
+            line = line_orig.rstrip()
+
+            # Strip leading spaces, due to C indent. Skip first pragma line.
+            if macro and leading_spaces is None:
+                leading_spaces = len(line) - len(line.lstrip())
+
+            # check for start of macro
+            if not macro:
+                leading_spaces = None
+                if result := re.match(PATTERN_PRAGMA, line):
+                    macro = Macro(linenr, type=result['pragma'])
+                    line = result['rest'] # strip pragma prefix
+
+            if macro:
+                if leading_spaces:
+                    line = re.sub(f'^\s{{1,{leading_spaces}}}', '', line)
+                if more_lines := line.endswith('\\'):
+                    line = line[:-1]
+                if statement := Statement.parse(line, strip=strip, linenr=linenr):
+                    macro.push_line(statement)
+                if more_lines:
+                    continue # stay in macro
+                else:
+                    yield macro.render_call()
+                    yield '#line {}\n'.format(linenr + 1)
+                    macros.append(macro)
+                    macro = None
+            else:
+                yield line_orig
+
+        if macros:
+            yield '\n// Auto-generated template functions:'
+        for macro in macros:
+            yield macro.render_fun()
diff --git a/tools/glsl_preproc/main.py b/tools/glsl_preproc/main.py
new file mode 100755
index 0000000..fcfea1f
--- /dev/null
+++ b/tools/glsl_preproc/main.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+import sys
+import argparse
+
+from macros import Macro
+
+parser = argparse.ArgumentParser()
+parser.add_argument('input')
+parser.add_argument('output')
+parser.add_argument('-s', '--strip', default=False, action='store_true')
+args = parser.parse_args()
+
+with open(args.input) as infile:
+    with open(args.output, 'w') as outfile:
+        for line in Macro.process_file(infile, strip=args.strip):
+            outfile.write(line)
diff --git a/tools/glsl_preproc/meson.build b/tools/glsl_preproc/meson.build
new file mode 100644
index 0000000..677ef7c
--- /dev/null
+++ b/tools/glsl_preproc/meson.build
@@ -0,0 +1,13 @@
+strip_arg = get_option('debug') ? [] : [ '--strip' ]
+glsl_preproc = [ python, join_paths(meson.current_source_dir(), 'main.py') ] + \
+                 strip_arg + [ '@INPUT@', '@OUTPUT@' ]
+glsl_deps = files(
+  'macros.py',
+  'statement.py',
+  'templates.py',
+  'templates/call.c.j2',
+  'templates/function.c.j2',
+  'templates/glsl_block.c.j2',
+  'templates/struct.c.j2',
+  'variables.py',
+)
diff --git a/tools/glsl_preproc/statement.py b/tools/glsl_preproc/statement.py
new file mode 100644
index 0000000..8641e94
--- /dev/null
+++ b/tools/glsl_preproc/statement.py
@@ -0,0 +1,301 @@
+import re
+
+from templates import GLSL_BLOCK_TEMPLATE
+from variables import VarSet, slugify
+
+VAR_PATTERN = re.compile(flags=re.VERBOSE, pattern=r'''
+    # long form ${ ... } syntax
+    \${ (?:\s*(?P<type>(?:              # optional type prefix
+            ident                       # identifiers (always dynamic)
+          | (?:(?:const|dynamic)\s+)?   # optional const/dynamic modifiers
+                (?:float|u?int)         # base type
+          | swizzle                     # swizzle mask
+          | (?:i|u)?vecType             # vector type (for mask)
+        )):)?
+        (?P<expr>[^{}]+)
+      }
+|   \$(?P<name>\w+) # reference to captured variable
+|   @(?P<var>\w+)   # reference to locally defined var
+''')
+
+class FmtSpec(object):
+    def __init__(self, ctype='ident_t', fmtstr='_%hx',
+                 wrap_expr=lambda name, expr: expr,
+                 fmt_expr=lambda name: name):
+        self.ctype     = ctype
+        self.fmtstr    = fmtstr
+        self.wrap_expr = wrap_expr
+        self.fmt_expr  = fmt_expr
+
+    @staticmethod
+    def wrap_var(type, dynamic=False):
+        if dynamic:
+            return lambda name, expr: f'sh_var_{type}(sh, "{name}", {expr}, true)'
+        else:
+            return lambda name, expr: f'sh_const_{type}(sh, "{name}", {expr})'
+
+    @staticmethod
+    def wrap_fn(fn):
+        return lambda name: f'{fn}({name})'
+
+VAR_TYPES = {
+    # identifiers: get mapped as-is
+    'ident':            FmtSpec(),
+
+    # normal variables: get mapped as shader constants
+    'int':              FmtSpec(wrap_expr=FmtSpec.wrap_var('int')),
+    'uint':             FmtSpec(wrap_expr=FmtSpec.wrap_var('uint')),
+    'float':            FmtSpec(wrap_expr=FmtSpec.wrap_var('float')),
+
+    # constant variables: get printed directly into the source code
+    'const int':        FmtSpec(ctype='int',          fmtstr='%d'),
+    'const uint':       FmtSpec(ctype='unsigned',     fmtstr='uint(%u)'),
+    'const float':      FmtSpec(ctype='float',        fmtstr='float(%f)'),
+
+    # dynamic variables: get loaded as shader variables
+    'dynamic int':      FmtSpec(wrap_expr=FmtSpec.wrap_var('int', dynamic=True)),
+    'dynamic uint':     FmtSpec(wrap_expr=FmtSpec.wrap_var('uint', dynamic=True)),
+    'dynamic float':    FmtSpec(wrap_expr=FmtSpec.wrap_var('float', dynamic=True)),
+
+    # component mask types
+    'swizzle':          FmtSpec(ctype='uint8_t', fmtstr='%s', fmt_expr=FmtSpec.wrap_fn('sh_swizzle')),
+    'ivecType':         FmtSpec(ctype='uint8_t', fmtstr='%s', fmt_expr=FmtSpec.wrap_fn('sh_float_type')),
+    'uvecType':         FmtSpec(ctype='uint8_t', fmtstr='%s', fmt_expr=FmtSpec.wrap_fn('sh_float_type')),
+    'vecType':          FmtSpec(ctype='uint8_t', fmtstr='%s', fmt_expr=FmtSpec.wrap_fn('sh_float_type')),
+}
+
+def stringify(value, strip):
+    end = '\\n"'
+    if strip:
+        end = '"'
+        value = re.sub(r'(?:\/\*[^\*]*\*\/|\/\/[^\n]+|^\s*)', '', value)
+    return '"' + value.replace('\\', '\\\\').replace('"', '\\"') + end
+
+def commentify(value, strip):
+    if strip:
+        return ''
+    return '/*' + value.replace('/*', '[[').replace('*/', ']]') + '*/'
+
+# Represents a statement + its enclosed variables
+class Statement(object):
+    def __init__(self, linenr=0):
+        super().__init__()
+        self.linenr = linenr
+        self.vars = VarSet()
+
+    def add_var(self, ctype, expr, name=None):
+        return self.vars.add_var(ctype, expr, name, self.linenr)
+
+    def render(self):
+        raise NotImplementedError
+
+    @staticmethod
+    def parse(text_orig, **kwargs):
+        raise NotImplementedError
+
+# Represents a single line of GLSL
+class GLSLLine(Statement):
+    class GLSLVar(object): # variable reference
+        def __init__(self, fmt, var):
+            self.fmt = fmt
+            self.var = var
+
+    def __init__(self, text, strip=False, **kwargs):
+        super().__init__(**kwargs)
+        self.refs = []
+        self.strip = strip
+
+        # produce two versions of line, one for printf() and one for append()
+        text = text.rstrip()
+        self.rawstr = stringify(text, strip)
+        self.fmtstr = stringify(re.sub(VAR_PATTERN, self.handle_var, text.replace('%', '%%')), strip)
+
+    def handle_var(self, match):
+        # local @var
+        if match['var']:
+            self.refs.append(match['var'])
+            return '%d'
+
+        # captured $var
+        type = match['type']
+        name = match['name']
+        expr = match['expr'] or name
+        name = name or slugify(expr)
+
+        fmt = VAR_TYPES[type or 'ident']
+        self.refs.append(fmt.fmt_expr(self.add_var(
+            ctype = fmt.ctype,
+            expr  = fmt.wrap_expr(name, expr),
+            name  = name,
+        )))
+
+        if fmt.ctype == 'ident_t':
+            return commentify(name, self.strip) + fmt.fmtstr
+        else:
+            return fmt.fmtstr
+
+# Represents an entire GLSL block
+class GLSLBlock(Statement):
+    def __init__(self, line):
+        super().__init__(linenr=line.linenr)
+        self.lines = []
+        self.refs  = []
+        self.append(line)
+
+    def append(self, line):
+        assert isinstance(line, GLSLLine)
+        self.lines.append(line)
+        self.refs += line.refs
+        self.vars.merge(line.vars)
+
+    def render(self):
+        return GLSL_BLOCK_TEMPLATE.render(block=self)
+
+# Represents a statement which can either take a single line or a block
+class BlockStart(Statement):
+    def __init__(self, multiline=False, **kwargs):
+        super().__init__(**kwargs)
+        self.multiline = multiline
+
+    def add_brace(self, text):
+        if self.multiline:
+            text += ' {'
+        return text
+
+# Represents an @if
+class IfCond(BlockStart):
+    def __init__(self, cond, inner=False, **kwargs):
+        super().__init__(**kwargs)
+        self.cond = cond if inner else self.add_var('bool', expr=cond)
+
+    def render(self):
+        return self.add_brace(f'if ({self.cond})')
+
+# Represents an @else
+class Else(BlockStart):
+    def __init__(self, closing, **kwargs):
+        super().__init__(**kwargs)
+        self.closing = closing
+
+    def render(self):
+        text = '} else' if self.closing else 'else'
+        return self.add_brace(text)
+
+# Represents a normal (integer) @for loop, or an (unsigned 8-bit) bitmask loop
+class ForLoop(BlockStart):
+    def __init__(self, var, op, bound, **kwargs):
+        super().__init__(**kwargs)
+        self.comps = op == ':'
+        self.bound = self.add_var('uint8_t' if self.comps else 'int', expr=bound)
+        self.var   = var
+        self.op    = op
+
+    def render(self):
+        if self.comps:
+            loopstart = f'uint8_t _mask = {self.bound}, {self.var}'
+            loopcond  = f'_mask && ({self.var} = __builtin_ctz(_mask), 1)'
+            loopstep  = f'_mask &= ~(1u << {self.var})'
+        else:
+            loopstart = f'int {self.var} = 0'
+            loopcond  = f'{self.var} {self.op} {self.bound}'
+            loopstep  = f'{self.var}++'
+
+        return self.add_brace(f'for ({loopstart}; {loopcond}; {loopstep})')
+
+# Represents a @switch block
+class Switch(Statement):
+    def __init__(self, expr, **kwargs):
+        super().__init__(**kwargs)
+        self.expr = self.add_var('unsigned', expr=expr)
+
+    def render(self):
+        return f'switch ({self.expr}) {{'
+
+# Represents a @case label
+class Case(Statement):
+    def __init__(self, label, **kwargs):
+        super().__init__(**kwargs)
+        self.label = label
+
+    def render(self):
+        return f'case {self.label}:'
+
+# Represents a @default line
+class Default(Statement):
+    def render(self):
+        return 'default:'
+
+# Represents a @break line
+class Break(Statement):
+    def render(self):
+        return 'break;'
+
+# Represents a single closing brace
+class EndBrace(Statement):
+    def render(self):
+        return '}'
+
+# Shitty regex-based statement parser
+PATTERN_IF  = re.compile(flags=re.VERBOSE, pattern=r'''
+@\s*if\s*                       # '@if'
+(?P<inner>@)?                   # optional leading @
+\((?P<cond>.+)\)\s*             # (condition)
+(?P<multiline>{)?\s*            # optional trailing {
+$''')
+
+PATTERN_ELSE = re.compile(flags=re.VERBOSE, pattern=r'''
+@\s*(?P<closing>})?\s*          # optional leading }
+else\s*                         # 'else'
+(?P<multiline>{)?\s*            # optional trailing {
+$''')
+
+PATTERN_FOR = re.compile(flags=re.VERBOSE, pattern=r'''
+@\s*for\s+\(                    # '@for' (
+(?P<var>\w+)\s*                 # loop variable name
+(?P<op>(?:\<=?|:))(?=[\w\s])\s* # '<', '<=' or ':', followed by \s or \w
+(?P<bound>[^\s].*)\s*           # loop boundary expression
+\)\s*(?P<multiline>{)?\s*       # ) and optional trailing {
+$''')
+
+PATTERN_SWITCH = re.compile(flags=re.VERBOSE, pattern=r'''
+@\s*switch\s*                   # '@switch'
+\((?P<expr>.+)\)\s*{            # switch expression
+$''')
+
+PATTERN_CASE = re.compile(flags=re.VERBOSE, pattern=r'''
+@\s*case\s*                     # '@case'
+(?P<label>[^:]+):?              # case label, optionally followed by :
+$''')
+
+PATTERN_BREAK   = r'@\s*break;?\s*$'
+PATTERN_DEFAULT = r'@\s*default:?\s*$'
+PATTERN_BRACE   = r'@\s*}\s*$'
+
+PARSERS = {
+    PATTERN_IF:         lambda r, **kw: IfCond(r['cond'], inner=r['inner'], multiline=r['multiline'], **kw),
+    PATTERN_ELSE:       lambda r, **kw: Else(closing=r['closing'], multiline=r['multiline'], **kw),
+    PATTERN_FOR:        lambda r, **kw: ForLoop(r['var'], r['op'], r['bound'], multiline=r['multiline'], **kw),
+    PATTERN_SWITCH:     lambda r, **kw: Switch(r['expr'], **kw),
+    PATTERN_CASE:       lambda r, **kw: Case(r['label'], **kw),
+    PATTERN_BREAK:      lambda _, **kw: Break(**kw),
+    PATTERN_DEFAULT:    lambda _, **kw: Default(**kw),
+    PATTERN_BRACE:      lambda _, **kw: EndBrace(**kw),
+}
+
+def parse_line(text_orig, strip, **kwargs):
+    # skip empty lines
+    text = text_orig.strip()
+    if not text:
+        return None
+    if text.lstrip().startswith('@'):
+        # try parsing as statement
+        for pat, fun in PARSERS.items():
+            if res := re.match(pat, text):
+                return fun(res, **kwargs)
+        # return generic error for unrecognized statements
+        raise SyntaxError('Syntax error in directive: ' + text.lstrip())
+    else:
+        # default to literal GLSL line
+        return GLSLLine(text_orig, strip, **kwargs)
+
+Statement.parse = parse_line
diff --git a/tools/glsl_preproc/templates.py b/tools/glsl_preproc/templates.py
new file mode 100644
index 0000000..b3b6c44
--- /dev/null
+++ b/tools/glsl_preproc/templates.py
@@ -0,0 +1,14 @@
+import jinja2
+import os.path
+
+TEMPLATEDIR = os.path.dirname(__file__) + '/templates'
+TEMPLATES = jinja2.Environment(
+    loader        = jinja2.FileSystemLoader(searchpath=TEMPLATEDIR),
+    lstrip_blocks = True,
+    trim_blocks   = True,
+)
+
+GLSL_BLOCK_TEMPLATE = TEMPLATES.get_template('glsl_block.c.j2')
+FUNCTION_TEMPLATE   = TEMPLATES.get_template('function.c.j2')
+CALL_TEMPLATE       = TEMPLATES.get_template('call.c.j2')
+STRUCT_TEMPLATE     = TEMPLATES.get_template('struct.c.j2')
diff --git a/tools/glsl_preproc/templates/call.c.j2 b/tools/glsl_preproc/templates/call.c.j2
new file mode 100644
index 0000000..61ee6c0
--- /dev/null
+++ b/tools/glsl_preproc/templates/call.c.j2
@@ -0,0 +1,19 @@
+{
+{% if macro.vars %}
+    const {{ macro.render_struct() }} {{ macro.name }}_args = {
+    {% for var in macro.vars %}
+#line {{ var.linenr }}
+        .{{ var.name }} = {{ var.expr }},
+    {% endfor %}
+    };
+#line {{ macro.linenr }}
+{% endif %}
+    size_t {{ macro.name }}_fn(void *, pl_str *, const uint8_t *);
+{% if macro.vars %}
+    pl_str_builder_append(sh->buffers[{{ macro.buf }}], {{ macro.name }}_fn,
+                          &{{ macro.name }}_args, sizeof({{ macro.name }}_args));
+{% else %}
+    pl_str_builder_append(sh->buffers[{{ macro.buf }}], {{ macro.name }}_fn, NULL, 0);
+{% endif %}
+}
+
diff --git a/tools/glsl_preproc/templates/function.c.j2 b/tools/glsl_preproc/templates/function.c.j2
new file mode 100644
index 0000000..9216472
--- /dev/null
+++ b/tools/glsl_preproc/templates/function.c.j2
@@ -0,0 +1,19 @@
+
+size_t {{ macro.name }}_fn(void *alloc, pl_str *buf, const uint8_t *ptr);
+size_t {{ macro.name }}_fn(void *alloc, pl_str *buf, const uint8_t *ptr)
+{
+{% if macro.vars %}
+{{ macro.render_struct() }} {{ Var.STRUCT_NAME }};
+memcpy(&{{ Var.STRUCT_NAME }}, ptr, sizeof({{ Var.STRUCT_NAME }}));
+{% endif %}
+
+{% for statement in macro.body %}
+{{ statement.render() }}
+{% endfor %}
+
+{% if macro.vars %}
+return sizeof({{ Var.STRUCT_NAME }});
+{% else %}
+return 0;
+{% endif %}
+}
diff --git a/tools/glsl_preproc/templates/glsl_block.c.j2 b/tools/glsl_preproc/templates/glsl_block.c.j2
new file mode 100644
index 0000000..aa8372d
--- /dev/null
+++ b/tools/glsl_preproc/templates/glsl_block.c.j2
@@ -0,0 +1,17 @@
+#line {{ block.linenr }}
+{% if block.refs %}
+    pl_str_append_asprintf_c(alloc, buf,
+    {% for line in block.lines %}
+        {{ line.fmtstr }}{{ ',' if loop.last }}
+    {% endfor %}
+    {% for ref in block.refs %}
+        {{ ref }}{{ ',' if not loop.last }}
+    {% endfor %}
+    );
+{% else %}
+    pl_str_append(alloc, buf, pl_str0(
+    {% for line in block.lines %}
+        {{ line.rawstr }}
+    {% endfor %}
+    ));
+{% endif %}
diff --git a/tools/glsl_preproc/templates/struct.c.j2 b/tools/glsl_preproc/templates/struct.c.j2
new file mode 100644
index 0000000..6a6a8fb
--- /dev/null
+++ b/tools/glsl_preproc/templates/struct.c.j2
@@ -0,0 +1,5 @@
+struct __attribute__((__packed__)) {
+{% for var in macro.vars %}
+    {{ var.ctype }} {{ var.name }};
+{% endfor %}
+}
diff --git a/tools/glsl_preproc/variables.py b/tools/glsl_preproc/variables.py
new file mode 100644
index 0000000..187fd79
--- /dev/null
+++ b/tools/glsl_preproc/variables.py
@@ -0,0 +1,79 @@
+import re
+
+def slugify(value):
+    value = re.sub(r'[^\w]+', '_', value.lower()).strip('_')
+    if value[:1].isdigit():
+        value = '_' + value
+    return value
+
+# A single variable (enclosed by the template)
+class Var(object):
+    STRUCT_NAME = 'vars'
+    CSIZES = {
+        # This array doesn't have to be exact, it's only used for sorting
+        # struct members to save a few bytes of memory here and there
+        'int':      4,
+        'unsigned': 4,
+        'float':    4,
+        'ident_t':  2,
+        'uint8_t':  1,
+        'bool':     1,
+    }
+
+    def __init__(self, ctype, expr, name, csize=0, linenr=0):
+        self.ctype  = ctype
+        self.csize  = csize or Var.CSIZES[ctype]
+        self.expr   = expr
+        self.name   = name
+        self.linenr = linenr
+
+    def __str__(self):
+        return f'{Var.STRUCT_NAME}.{self.name}'
+
+def is_literal(expr):
+    return expr.isnumeric() or expr in ['true', 'false']
+
+# A (deduplicated) set of variables
+class VarSet(object):
+    def __init__(self):
+        self.varmap = {} # expr -> cvar
+
+    def __iter__(self):
+        # Sort from largest to smallest variable to optimize struct padding
+        yield from sorted(self.varmap.values(),
+            reverse=True,
+            key=lambda v: v.csize,
+        )
+
+    def __bool__(self):
+        return True if self.varmap else False
+
+    def add_var_raw(self, var):
+        # Re-use existing entry for identical expression/type pairs
+        if old := self.varmap.get(var.expr):
+            if var.ctype != old.ctype:
+                raise SyntaxError(f'Conflicting types for expression {var.expr}, '
+                                  f'got {var.ctype}, expected {old.ctype}')
+            assert old.name == var.name
+            return old
+
+        names = [ v.name for v in self.varmap.values() ]
+        while var.name in names:
+            var.name += '_'
+        self.varmap[var.expr] = var
+        return var
+
+    # Returns the added variable
+    def add_var(self, ctype, expr, name=None, linenr=0):
+        assert expr
+        expr = expr.strip()
+        if is_literal(expr):
+            return expr
+        name = name or slugify(expr)
+
+        var = Var(ctype, expr=expr, name=name, linenr=linenr)
+        return self.add_var_raw(var)
+
+    def merge(self, other):
+        for var in other:
+            self.add_var_raw(var)