path: root/src/boost/tools/litre/cplusplus.py

# Copyright David Abrahams 2004. 
# Copyright Daniel Wallin 2006.
# Distributed under the Boost
# Software License, Version 1.0. (See accompanying
# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

import os
import tempfile
import litre
import re
import sys
import traceback

# Thanks to Jean Brouwers for this snippet
def _caller(up=0):
    '''Get file name, line number, function name and
       source text of the caller's caller as 4-tuple:
       (file, line, func, text).

       The optional argument 'up' allows retrieval of 
       a caller further back up into the call stack.

       Note, the source text may be None and function
       name may be '?' in the returned result.  In
       Python 2.3+ the file name may be an absolute
       path.
    '''
    try:  # just get a few frames'
        f = traceback.extract_stack(limit=up+2)
        if f:
           return f[0]
    except:
        pass
    # running with psyco?
    return ('', 0, '', None)

class Example:
    closed = False
    in_emph = None
    
    def __init__(self, node, section, line_offset, line_hash = '#'):
        # A list of text fragments comprising the Example.  Start with a #line
        # directive
        self.section = section
        self.line_hash = line_hash
        self.node = node
        self.body = []
        self.line_offset = line_offset
        self._number_of_prefixes = 0

        self.emphasized = [] # indices of text strings that have been
                             # emphasized.  These are generally expected to be
                             # invalid C++ and will need special treatment

    def begin_emphasis(self):
        self.in_emph = len(self.body)

    def end_emphasis(self):
        self.emphasized.append( (self.in_emph, len(self.body)) )
        
    def append(self, s):
        self.append_raw(self._make_line(s))

    def prepend(self, s):
        self.prepend_raw(self._make_line(s))

    def append_raw(self, s):
        self.body.append(s)

    def prepend_raw(self, s):
        self.body.insert(0,s)
        self.emphasized = [ (x[0]+1,x[1]+1) for x in self.emphasized ]
        self._number_of_prefixes += 1

    def replace(self, s1, s2):
        self.body = [x.replace(s1,s2) for x in self.body]
        
    def sub(self, pattern, repl, count = 1, flags = re.MULTILINE):
        pat = re.compile(pattern, flags)
        for i,txt in enumerate(self.body):
            if count > 0:
                x, subs = pat.subn(repl, txt, count)
                self.body[i] = x
                count -= subs
        
    def wrap(self, s1, s2):
        self.append_raw(self._make_line(s2))
        self.prepend_raw(self._make_line(s1, offset = -s1.count('\n')))
        
    def replace_emphasis(self, s, index = 0):
        """replace the index'th emphasized text with s"""
        e = self.emphasized[index]
        self.body[e[0]:e[1]] = [s]
        del self.emphasized[index]

    elipsis = re.compile('^([ \t]*)([.][.][.][ \t]*)$', re.MULTILINE)
    
    def __str__(self):
        # Comment out any remaining emphasized sections
        b = [self.elipsis.sub(r'\1// \2', s) for s in self.body]
        emph = self.emphasized
        emph.reverse()
        for e in emph:
            b.insert(e[1], ' */')
            b.insert(e[0], '/* ')
        emph.reverse()

        # Add initial #line
        b.insert(
            self._number_of_prefixes,
            self._line_directive(self.node.line, self.node.source)
            )

        # Add trailing newline to avoid warnings
        b.append('\n')
        return ''.join(b)

    def __repr__(self):
        return "Example: " + repr(str(self))

    def raw(self):
        return ''.join(self.body)

    def _make_line(self, s, offset = 0):
        c = _caller(2)[1::-1]
        offset -= s.count('\n')
        return '\n%s%s\n' % (self._line_directive(offset = offset, *c), s.strip('\n'))

    def _line_directive(self, line, source, offset = None):
        if self.line_hash is None:
            return '\n'
        
        if offset is None:
            offset = self.line_offset
            
        if line is None or line <= -offset:
            line = 1
        else:
            line += offset

        if source is None:
            return '%sline %d\n' % (self.line_hash, line)
        else:
            return '%sline %d "%s"\n' % (self.line_hash, line, source)


def syscmd(
      cmd
    , expect_error = False
    , input = None
    , max_output_lines = None
    ):

    # On windows close() returns the exit code, on *nix it doesn't so
    # we need to use popen2.Popen4 instead.
    if sys.platform == 'win32':
        stdin, stdout_stderr = os.popen4(cmd)
        if input: stdin.write(input)
        stdin.close()
        
        out = stdout_stderr.read()
        status = stdout_stderr.close()
    else:
        import popen2
        process = popen2.Popen4(cmd)
        if input: process.tochild.write(input)
        out = process.fromchild.read()
        status = process.wait()

    if max_output_lines is not None:
        out = '\n'.join(out.split('\n')[:max_output_lines])
        
    if expect_error:
        status = not status

    if status:
        print
        print '========== offending command ==========='
        print cmd
        print '------------ stdout/stderr -------------'
        print expect_error and 'Error expected, but none seen' or out
    elif expect_error > 1:
        print
        print '------ Output of Expected Error --------'
        print out
        print '----------------------------------------'
        
    sys.stdout.flush()
        
    return (status,out)


def expand_vars(path):
    if os.name == 'nt':
        re_env = re.compile(r'%\w+%')
        return re_env.sub(
              lambda m: os.environ.get( m.group(0)[1:-1] )
            , path
            )
    else:
        return os.path.expandvars(path)

def remove_directory_and_contents(path):
    for root, dirs, files in os.walk(path, topdown=False):
        for name in files:
            os.remove(os.path.join(root, name))
        for name in dirs:
            os.rmdir(os.path.join(root, name))
    os.rmdir(path)

class BuildResult:
    def __init__(self, path):
        self.path = path

    def __repr__(self):
        return self.path

    def __del__(self):
        remove_directory_and_contents(self.path)

class CPlusPlusTranslator(litre.LitreTranslator):

    _exposed_attrs = ['compile', 'test', 'ignore', 'match_stdout', 'stack', 'config'
                      , 'example', 'prefix', 'preprocessors', 'litre_directory',
                      'litre_translator', 'includes', 'build', 'jam_prefix',
                      'run_python']

    last_run_output = ''

    """Attributes that will be made available to litre code"""
    
    def __init__(self, document, config):
        litre.LitreTranslator.__init__(self, document, config)
        self.in_literal = False
        self.in_table = True
        self.preprocessors = []
        self.stack = []
        self.example = None
        self.prefix = []
        self.includes = config.includes
        self.litre_directory = os.path.split(__file__)[0]
        self.config = config
        self.litre_translator = self
        self.line_offset = 0
        self.last_source = None
        self.jam_prefix = []

        self.globals = { 'test_literals_in_tables' : False }
        for m in self._exposed_attrs:
            self.globals[m] = getattr(self, m)

        self.examples = {}
        self.current_section = None

    #
    # Stuff for use by docutils writer framework
    #
    def visit_emphasis(self, node):
        if self.in_literal:
            self.example.begin_emphasis()
        
    def depart_emphasis(self, node):
        if self.in_literal:
            self.example.end_emphasis()

    def visit_section(self, node):
        self.current_section = node['ids'][0]

    def visit_literal_block(self, node):
        if node.source is None:
            node.source = self.last_source
        self.last_source = node.source

        # create a new example
        self.example = Example(node, self.current_section, line_offset = self.line_offset, line_hash = self.config.line_hash)

        self.stack.append(self.example)

        self.in_literal = True

    def depart_literal_block(self, node):
        self.in_literal = False

    def visit_literal(self, node):
        if self.in_table and self.globals['test_literals_in_tables']:
            self.visit_literal_block(node)
        else:
            litre.LitreTranslator.visit_literal(self,node)
        
    def depart_literal(self, node):
        if self.in_table and self.globals['test_literals_in_tables']:
            self.depart_literal_block(node)
        else:
            litre.LitreTranslator.depart_literal(self,node)

    def visit_table(self,node):
        self.in_table = True
        litre.LitreTranslator.visit_table(self,node)
        
    def depart_table(self,node):
        self.in_table = False
        litre.LitreTranslator.depart_table(self,node)
        
    def visit_Text(self, node):
        if self.in_literal:
            self.example.append_raw(node.astext())

    def depart_document(self, node):
        self.write_examples()
           
    #
    # Private stuff
    #
            
    def handled(self, n = 1):
        r = self.stack[-n:]
        del self.stack[-n:]
        return r

    def _execute(self, code):
        """Override of litre._execute; sets up variable context before
        evaluating code
        """
        self.globals['example'] = self.example
        eval(code, self.globals)
        
    #
    # Stuff for use by embedded python code
    #

    def match_stdout(self, expected = None):

        if expected is None:
            expected = self.example.raw()
            self.handled()

        if not re.search(expected, self.last_run_output, re.MULTILINE):
        #if self.last_run_output.strip('\n') != expected.strip('\n'):
            print 'output failed to match example'
            print '-------- Actual Output -------------'
            print repr(self.last_run_output)
            print '-------- Expected Output -----------'
            print repr(expected)
            print '------------------------------------'
            sys.stdout.flush()
            
    def ignore(self, n = 1):
        if n == 'all':
            n = len(self.stack)
        return self.handled(n)

    def wrap(self, n, s1, s2):
        self.stack[-1].append(s2)
        self.stack[-n].prepend(s1)
        

    def compile(
          self
        , howmany = 1
        , pop = -1
        , expect_error = False
        , extension = '.o'
        , options = ['-c']
        , built_handler = lambda built_file: None
        , source_file = None
        , source_suffix = '.cpp'
          # C-style comments by default; handles C++ and YACC
        , make_comment = lambda text: '/*\n%s\n*/' % text 
        , built_file = None
        , command = None
        ):
        """
        Compile examples on the stack, whose topmost item is the last example
        seen but not yet handled so far.

        :howmany: How many of the topmost examples on the stack to compile.
           You can pass a number, or 'all' to indicate that all examples should
           be compiled.

        :pop: How many of the topmost examples to discard.  By default, all of
           the examples that are compiled are discarded.
        
        :expect_error: Whether a compilation error is to be expected.  Any value
           > 1 will cause the expected diagnostic's text to be dumped for
           diagnostic purposes.  It's common to expect an error but see a
           completely unrelated one because of bugs in the example (you can get
           this behavior for all examples by setting show_expected_error_output
           in your config).

        :extension: The extension of the file to build (set to .exe for
           run)

        :options: Compiler flags

        :built_file: A path to use for the built file.  By default, a temp
            filename is conjured up
            
        :built_handler: A function that's called with the name of the built file
           upon success.

        :source_file: The full name of the source file to write
        
        :source_suffix: If source_file is None, the suffix to use for the source file

        :make_comment: A function that transforms text into an appropriate comment.
        
        :command: A function that is passed (includes, opts, target, source), where
           opts is a string representing compiler options, target is the name of
           the file to build, and source is the name of the file into which the
           example code is written.  By default, the function formats
           litre.config.compiler with its argument tuple.
        """

        # Grab one example by default
        if howmany == 'all':
            howmany = len(self.stack)

        source = '\n'.join(
            self.prefix
            + [str(x) for x in self.stack[-howmany:]]
            )

        source = reduce(lambda s, f: f(s), self.preprocessors, source)
        
        if pop:
            if pop < 0:
                pop = howmany
            del self.stack[-pop:]
            
        if len(self.stack):
            self.example = self.stack[-1]

        cpp = self._source_file_path(source_file, source_suffix)
            
        if built_file is None:
            built_file = self._output_file_path(source_file, extension)
        
        opts = ' '.join(options)
        
        includes = ' '.join(['-I%s' % d for d in self.includes])
        if not command:
            command = self.config.compiler

        if type(command) == str:
            command = lambda i, o, t, s, c = command: c % (i, o, t, s)
            
        cmd = command(includes, opts, expand_vars(built_file), expand_vars(cpp))

        if expect_error and self.config.show_expected_error_output:
            expect_error += 1


        comment_cmd = command(includes, opts, built_file, os.path.basename(cpp))
        comment = make_comment(config.comment_text(comment_cmd, expect_error))

        self._write_source(cpp, '\n'.join([comment, source]))

        #print 'wrote in', cpp
        #print 'trying command', cmd

        status, output = syscmd(cmd, expect_error)

        if status or expect_error > 1:
            print
            if expect_error and expect_error < 2:
                print 'Compilation failure expected, but none seen'
            print '------------ begin offending source ------------'
            print open(cpp).read()
            print '------------ end offending source ------------'
            
            if self.config.save_cpp:
                print 'saved in', repr(cpp)
            else:
                self._remove_source(cpp)

            sys.stdout.flush()
        else:
            print '.',
            sys.stdout.flush()
            built_handler(built_file)
                
            self._remove_source(cpp)
            
        try:
            self._unlink(built_file)
        except:
            if not expect_error:
                print 'failed to unlink', built_file
        
        return status

    def test(
          self
        , rule = 'run'
        , howmany = 1
        , pop = -1
        , expect_error = False
        , requirements = ''
        , input = ''
        ):

        # Grab one example by default
        if howmany == 'all':
            howmany = len(self.stack)

        source = '\n'.join(
            self.prefix
            + [str(x) for x in self.stack[-howmany:]]
            )

        source = reduce(lambda s, f: f(s), self.preprocessors, source)

        id = self.example.section
        if not id:
            id = 'top-level'

        if not self.examples.has_key(self.example.section):
            self.examples[id] = [(rule, source)]
        else:
            self.examples[id].append((rule, source))

        if pop:
            if pop < 0:
                pop = howmany
            del self.stack[-pop:]

        if len(self.stack):
            self.example = self.stack[-1]

    def write_examples(self):
        jam = open(os.path.join(self.config.dump_dir, 'Jamfile.v2'), 'w')

        jam.write('''
import testing ;

''')

        for id,examples in self.examples.items():
            for i in range(len(examples)):
                cpp = '%s%d.cpp' % (id, i)

                jam.write('%s %s ;\n' % (examples[i][0], cpp))

                outfile = os.path.join(self.config.dump_dir, cpp)
                print cpp,
                try:
                    if open(outfile, 'r').read() == examples[i][1]:
                        print ' .. skip'
                        continue
                except:
                    pass

                open(outfile, 'w').write(examples[i][1])
                print ' .. written'

        jam.close()

    def build(
          self
        , howmany = 1
        , pop = -1
        , source_file = 'example.cpp'
        , expect_error = False
        , target_rule = 'obj'
        , requirements = ''
        , input = ''
        , output = 'example_output'
        ):

        # Grab one example by default
        if howmany == 'all':
            howmany = len(self.stack)

        source = '\n'.join(
            self.prefix
            + [str(x) for x in self.stack[-howmany:]]
            )

        source = reduce(lambda s, f: f(s), self.preprocessors, source)

        if pop:
            if pop < 0:
                pop = howmany
            del self.stack[-pop:]

        if len(self.stack):
            self.example = self.stack[-1]

        dir = tempfile.mkdtemp()
        cpp = os.path.join(dir, source_file)
        self._write_source(cpp, source)
        self._write_jamfile(
            dir
          , target_rule = target_rule
          , requirements = requirements
          , input = input
          , output = output
          )

        cmd = 'bjam'
        if self.config.bjam_options:
            cmd += ' %s' % self.config.bjam_options

        os.chdir(dir)
        status, output = syscmd(cmd, expect_error)

        if status or expect_error > 1:
            print
            if expect_error and expect_error < 2:
                print 'Compilation failure expected, but none seen'
            print '------------ begin offending source ------------'
            print open(cpp).read()
            print '------------ begin offending Jamfile -----------'
            print open(os.path.join(dir, 'Jamroot')).read()
            print '------------ end offending Jamfile -------------'

            sys.stdout.flush()
        else:
            print '.',
            sys.stdout.flush()

        if status: return None
        else: return BuildResult(dir)

    def _write_jamfile(self, path, target_rule, requirements, input, output):
        jamfile = open(os.path.join(path, 'Jamroot'), 'w')
        contents = r"""
import modules ;

BOOST_ROOT = [ modules.peek : BOOST_ROOT ] ;
use-project /boost : $(BOOST_ROOT) ;

%s

%s %s
  : example.cpp %s
  : <include>.
    %s
    %s
  ;
        """ % (
            '\n'.join(self.jam_prefix)
          , target_rule
          , output
          , input
          , ' '.join(['<include>%s' % d for d in self.includes])
          , requirements
          )

        jamfile.write(contents)

    def run_python(
          self
        , howmany = 1
        , pop = -1
        , module_path = []
        , expect_error = False
        ):
        # Grab one example by default
        if howmany == 'all':
            howmany = len(self.stack)

        if module_path == None: module_path = []

        if isinstance(module_path, BuildResult) or type(module_path) == str:
            module_path = [module_path]

        module_path = map(lambda p: str(p), module_path)

        source = '\n'.join(
            self.prefix
            + [str(x) for x in self.stack[-howmany:]]
            )

        if pop:
            if pop < 0:
                pop = howmany
            del self.stack[-pop:]

        if len(self.stack):
            self.example = self.stack[-1]

        r = re.compile(r'^(>>>|\.\.\.) (.*)$', re.MULTILINE)
        source = r.sub(r'\2', source)
        py = self._source_file_path(source_file = None, source_suffix = 'py')
        open(py, 'w').write(source)

        old_path = os.getenv('PYTHONPATH')
        if old_path == None:
            pythonpath = ':'.join(module_path)
            old_path = ''
        else:
            pythonpath = old_path + ':%s' % ':'.join(module_path)

        os.putenv('PYTHONPATH', pythonpath)
        status, output = syscmd('python %s' % py)

        if status or expect_error > 1:
            print
            if expect_error and expect_error < 2:
                print 'Compilation failure expected, but none seen'
            print '------------ begin offending source ------------'
            print open(py).read()
            print '------------ end offending Jamfile -------------'

            sys.stdout.flush()
        else:
            print '.',
            sys.stdout.flush()

        self.last_run_output = output
        os.putenv('PYTHONPATH', old_path)
        self._unlink(py)

    def _write_source(self, filename, contents):
        open(filename,'w').write(contents)
        
    def _remove_source(self, source_path):
        os.unlink(source_path)

    def _source_file_path(self, source_file, source_suffix):
        if source_file is None:
            cpp = tempfile.mktemp(suffix=source_suffix)
        else:
            cpp = os.path.join(tempfile.gettempdir(), source_file)
        return cpp
    
    def _output_file_path(self, source_file, extension):
        return tempfile.mktemp(suffix=extension)

    def _unlink(self, file):
        file = expand_vars(file)
        if os.path.exists(file):
            os.unlink(file)
    
    def _launch(self, exe, stdin = None):
        status, output = syscmd(exe, input = stdin)
        self.last_run_output = output
        
    def run_(self, howmany = 1, stdin = None, **kw):
        new_kw = { 'options':[], 'extension':'.exe' }
        new_kw.update(kw)

        self.compile(
            howmany
          , built_handler = lambda exe: self._launch(exe, stdin = stdin)
          , **new_kw
        )

    def astext(self):
        return ""
        return '\n\n ---------------- Unhandled Fragment ------------ \n\n'.join(
            [''] # generates a leading announcement
            + [ unicode(s) for s in self.stack]
            )

class DumpTranslator(CPlusPlusTranslator):
    example_index = 1
    
    def _source_file_path(self, source_file, source_suffix):
        if source_file is None:
            source_file = 'example%s%s' % (self.example_index, source_suffix)
            self.example_index += 1
            
        cpp = os.path.join(config.dump_dir, source_file)
        return cpp

    def _output_file_path(self, source_file, extension):
        chapter = os.path.basename(config.dump_dir)
        return '%%TEMP%%\metaprogram-%s-example%s%s' \
            % ( chapter, self.example_index - 1, extension)

    def _remove_source(self, source_path):
        pass


class WorkaroundTranslator(DumpTranslator):
    """Translator used to test/dump workaround examples for vc6 and vc7.  Just
    like a DumpTranslator except that we leave existing files alone.

    Warning: not sensitive to changes in .rst source!!  If you change the actual
    examples in source files you will have to move the example files out of the
    way and regenerate them, then re-incorporate the workarounds.
    """
    def _write_source(self, filename, contents):
        if not os.path.exists(filename):
            DumpTranslator._write_source(self, filename, contents)
    
class Config:
    save_cpp = False
    line_hash = '#'
    show_expected_error_output = False
    max_output_lines = None

class Writer(litre.Writer):
    translator = CPlusPlusTranslator
    
    def __init__(
        self
      , config
    ):
        litre.Writer.__init__(self)
        self._config = Config()
        defaults = Config.__dict__

        # update config elements
        self._config.__dict__.update(config.__dict__)
#             dict([i for i in config.__dict__.items()
#                   if i[0] in config.__all__]))