Adding upstream version 14.2.21.upstream/14.2.21upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

39 files changed, 4232 insertions, 0 deletions

diff --git a/src/boost/libs/spirit/test/lex/Jamfile b/src/boost/libs/spirit/test/lex/Jamfile
new file mode 100644
index 00000000..250656f1
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/Jamfile
@@ -0,0 +1,101 @@
+#==============================================================================
+#   Copyright (c) 2001-2011 Joel de Guzman
+#   Copyright (c) 2001-2012 Hartmut Kaiser
+#   Copyright (c)      2011 Bryce Lelbach
+#   Copyright (c) 2016-2019 Nikita Kniazev
+#
+#   Use, modification and distribution is subject to the Boost Software
+#   License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+#   http://www.boost.org/LICENSE_1_0.txt)
+#==============================================================================
+
+import modules ;
+import path ;
+import testing ;
+
+###############################################################################
+
+project spirit-lex
+    : requirements
+        <include>.
+        <c++-template-depth>512
+    ;
+
+###############################################################################
+
+cpp-pch pch : pch.hpp : : : <include>. <toolset>msvc:<cxxflags>"/FIpch.hpp" ;
+
+explicit pch ;
+
+###############################################################################
+
+local subproject-name = lex ;
+
+rule run ( sources + : args * : input-files *
+         : requirements * : target-name ? : default-build * )
+{
+    target-name ?= $(subproject-name)_$(sources[1]:D=:S=) ;
+    return [ testing.run $(sources) : $(args) : $(input-files)
+           : $(requirements) <pch>on-spirit:<source>pch : $(target-name) : $(default-build) ] ;
+}
+
+rule compile ( sources + : requirements * : target-name ? )
+{
+    target-name ?= $(subproject-name)_$(sources[1]:D=:S=) ;
+    return [ testing.compile $(sources)
+           : $(requirements) <pch>on-spirit:<source>pch : $(target-name) ] ;
+}
+
+rule compile-fail ( sources + : requirements * : target-name ? )
+{
+    target-name ?= $(subproject-name)_$(sources[1]:D=:S=) ;
+    return [ testing.compile-fail $(sources)
+           : $(requirements) <pch>on-spirit:<source>pch : $(target-name) ] ;
+}
+
+###############################################################################
+
+rule location ( name )		
+{		
+    local this = [ modules.binding $(__name__) ] ;		
+    local here = [ path.parent [ path.make $(this) ] ] ;		
+    return [ path.join $(here) $(name) ] ;		
+}		
+		
+###############################################################################
+
+run auto_switch_lexerstate.cpp ;
+run dedent_handling_phoenix.cpp ;
+run id_type_enum.cpp ;
+run lexertl1.cpp ;
+run lexertl2.cpp ;
+run lexertl3.cpp ;
+run lexertl4.cpp ;
+run lexertl5.cpp ;
+run lexer_state_switcher.cpp ;
+run semantic_actions.cpp ;
+run set_token_value.cpp ;
+run set_token_value_phoenix.cpp ;
+run state_switcher.cpp ;
+run string_token_id.cpp ;
+run token_iterpair.cpp ;
+run token_moretypes.cpp ;
+run token_omit.cpp ;
+run token_onetype.cpp ;
+run plain_token.cpp ;
+
+run regression_basic_lexer.cpp ;
+run regression_matlib_dynamic.cpp ;
+run regression_matlib_generate.cpp         : [ location matlib_static.h ] ;
+run regression_matlib_static.cpp           : : : <dependency>lex_regression_matlib_generate ;
+run regression_matlib_generate_switch.cpp  : [ location matlib_static_switch.h ] ;
+run regression_matlib_switch.cpp           : : : <dependency>lex_regression_matlib_generate_switch ;
+run regression_word_count.cpp ;
+run regression_syntax_error.cpp ;
+run regression_wide.cpp ;
+run regression_file_iterator1.cpp ;
+run regression_file_iterator2.cpp ;
+run regression_file_iterator3.cpp : : : <pch>off ;
+run regression_file_iterator4.cpp ;
+run regression_static_wide_6253.cpp ;
+run regression_less_8563.cpp ;
diff --git a/src/boost/libs/spirit/test/lex/auto_switch_lexerstate.cpp b/src/boost/libs/spirit/test/lex/auto_switch_lexerstate.cpp
new file mode 100644
index 00000000..6ba8bd86
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/auto_switch_lexerstate.cpp
@@ -0,0 +1,89 @@
+//  Copyright (c) 2001-2010 Hartmut Kaiser
+//  Copyright (c) 2010 Mathias Gaunard
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+// This test makes sure that the BOL state (begin of line) is properly reset
+// if a token matched at the beginning of a line is discarded using 
+// lex::pass_fail.
+// Additionally this test makes sure the syntax 'self("state", "targetstate")'
+// works properly.
+
+#include <boost/config/warning_disable.hpp>
+#include <boost/detail/lightweight_test.hpp>
+
+#include <boost/spirit/include/support_multi_pass.hpp>
+#include <boost/spirit/include/classic_position_iterator.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+
+namespace spirit = boost::spirit;
+namespace lex = spirit::lex;
+
+typedef spirit::classic::position_iterator2<
+    spirit::multi_pass<std::istreambuf_iterator<char> >
+> file_iterator;
+
+inline file_iterator 
+make_file_iterator(std::istream& input, const std::string& filename)
+{
+    return file_iterator(
+        spirit::make_default_multi_pass(
+            std::istreambuf_iterator<char>(input)),
+        spirit::multi_pass<std::istreambuf_iterator<char> >(),
+        filename);
+}
+
+typedef lex::lexertl::token<file_iterator> token_type;
+
+struct lexer
+  : lex::lexer<lex::lexertl::actor_lexer<token_type> >
+{
+    lexer() : word("^[a-zA-Z0-9]+$", 1)
+    {
+        self("INITIAL", "O") = 
+                word 
+            |   lex::string("!.*$") [ 
+                    lex::_pass = lex::pass_flags::pass_ignore 
+                ]
+            |   lex::token_def<>('\n', 2) 
+            ;
+        
+        self("O", "INITIAL") = 
+                lex::string(".") [ 
+                    lex::_pass = lex::pass_flags::pass_fail 
+                ]
+            ;
+    }
+    
+    lex::token_def<> word;
+};
+
+typedef lexer::iterator_type token_iterator;
+
+int main()
+{
+    std::stringstream ss;
+    ss << "!foo\nbar\n!baz";
+    
+    file_iterator begin = make_file_iterator(ss, "SS");
+    file_iterator end;
+    
+    lexer l;
+    token_iterator begin2 = l.begin(begin, end);
+    token_iterator end2 = l.end();
+    
+    std::size_t test_data[] = { 2, 1, 2 };
+    std::size_t const test_data_size = sizeof(test_data)/sizeof(test_data[0]);
+
+    token_iterator it = begin2;
+    std::size_t i = 0;
+    for (/**/; it != end2 && i < test_data_size; ++it, ++i)
+    {
+        BOOST_TEST(it->id() == test_data[i]);
+    }
+    BOOST_TEST(it == end2);
+    BOOST_TEST(i == test_data_size);
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/dedent_handling_phoenix.cpp b/src/boost/libs/spirit/test/lex/dedent_handling_phoenix.cpp
new file mode 100644
index 00000000..adffb3ab
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/dedent_handling_phoenix.cpp
@@ -0,0 +1,98 @@
+//  Copyright (c) 2009 Carl Barron
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <iostream>
+#include <sstream>
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/spirit/include/lex.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/phoenix_core.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+#include <boost/spirit/include/phoenix_statement.hpp>
+
+namespace lex = boost::spirit::lex;
+namespace phoenix = boost::phoenix;
+
+///////////////////////////////////////////////////////////////////////////////
+template <typename Lexer>
+struct multi_tokens : lex::lexer<Lexer>
+{
+    int level;
+
+    multi_tokens() : level(0)
+    {
+        using lex::_state;
+        using lex::_start;
+        using lex::_end;
+        using lex::_pass;
+        using lex::pass_flags;
+
+        a = "A";
+        b = "B";
+        c = "C";
+        this->self = 
+                a [ ++phoenix::ref(level) ]
+            |   b
+            |   c [
+                      _state = "in_dedenting",
+                      _end = _start,
+                      _pass = pass_flags::pass_ignore
+                  ]
+            ;
+
+        d = ".";
+        this->self("in_dedenting") = 
+                d [ 
+                      if_(--phoenix::ref(level)) [ 
+                          _end = _start 
+                      ]
+                      .else_ [ 
+                          _state = "INITIAL"
+                      ]
+                  ]
+            ;
+    }
+
+    lex::token_def<> a, b, c, d;
+};
+
+struct dumper
+{
+    typedef bool result_type;
+
+    dumper(std::stringstream& strm) : strm(strm) {}
+
+    template <typename Token>
+    bool operator () (Token const &t)
+    {
+        strm << (char)(t.id() - lex::min_token_id + 'a');
+        return true;
+    }
+
+    std::stringstream& strm;
+
+    // silence MSVC warning C4512: assignment operator could not be generated
+    BOOST_DELETED_FUNCTION(dumper& operator= (dumper const&));
+};
+
+///////////////////////////////////////////////////////////////////////////////
+int main()
+{
+    typedef lex::lexertl::token<std::string::iterator> token_type;
+    typedef lex::lexertl::actor_lexer<token_type> base_lexer_type;
+    typedef multi_tokens<base_lexer_type> lexer_type;
+
+    std::string in("AAABBC");
+    std::string::iterator first(in.begin());
+    std::stringstream strm;
+
+    lexer_type the_lexer;
+    BOOST_TEST(lex::tokenize(first, in.end(), the_lexer, dumper(strm)));
+    BOOST_TEST(strm.str() == "aaabbddd");
+
+    return boost::report_errors();
+}
+
diff --git a/src/boost/libs/spirit/test/lex/id_type_enum.cpp b/src/boost/libs/spirit/test/lex/id_type_enum.cpp
new file mode 100644
index 00000000..b3a5cf72
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/id_type_enum.cpp
@@ -0,0 +1,93 @@
+//  Copyright (c) 2001-2010 Hartmut Kaiser
+//  Copyright (c) 2010 Mathias Gaunard
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/config/warning_disable.hpp>
+#include <boost/detail/lightweight_test.hpp>
+
+#include <boost/spirit/include/support_multi_pass.hpp>
+#include <boost/spirit/include/classic_position_iterator.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+
+namespace spirit = boost::spirit;
+namespace lex = spirit::lex;
+
+typedef spirit::classic::position_iterator2<
+    spirit::multi_pass<std::istreambuf_iterator<char> >
+> file_iterator;
+
+inline file_iterator 
+make_file_iterator(std::istream& input, const std::string& filename)
+{
+    return file_iterator(
+        spirit::make_default_multi_pass(
+            std::istreambuf_iterator<char>(input)),
+        spirit::multi_pass<std::istreambuf_iterator<char> >(),
+        filename);
+}
+
+enum token_id
+{
+    ID_WORD = lex::min_token_id + 1,
+    ID_EOL
+};
+
+typedef lex::lexertl::token<
+    file_iterator, boost::mpl::vector<>, boost::mpl::true_, token_id
+> token_type;
+
+struct lexer
+  : lex::lexer<lex::lexertl::actor_lexer<token_type> >
+{
+    lexer() : word("^[a-zA-Z0-9]+$", ID_WORD)
+    {
+        typedef lex::token_def<lex::unused_type, char, token_id> toked_def;
+
+        self("INITIAL", "O") = 
+                word 
+            |   toked_def("!.*$") [ 
+                    lex::_pass = lex::pass_flags::pass_ignore 
+                ]
+            |   toked_def('\n', ID_EOL) 
+            ;
+        
+        self("O", "INITIAL") = 
+                toked_def(".") [ 
+                    lex::_pass = lex::pass_flags::pass_fail 
+                ]
+            ;
+    }
+    
+    lex::token_def<lex::unused_type, char, token_id> word;
+};
+
+typedef lexer::iterator_type token_iterator;
+
+int main()
+{
+    std::stringstream ss;
+    ss << "!foo\nbar\n!baz";
+    
+    file_iterator begin = make_file_iterator(ss, "SS");
+    file_iterator end;
+    
+    lexer l;
+    token_iterator begin2 = l.begin(begin, end);
+    token_iterator end2 = l.end();
+    
+    token_id test_data[] = { ID_EOL, ID_WORD, ID_EOL };
+    std::size_t const test_data_size = sizeof(test_data)/sizeof(test_data[0]);
+
+    token_iterator it = begin2;
+    std::size_t i = 0;
+    for (/**/; it != end2 && i < test_data_size; ++it, ++i)
+    {
+        BOOST_TEST(it->id() == test_data[i]);
+    }
+    BOOST_TEST(it == end2);
+    BOOST_TEST(i == test_data_size);
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/lexer_state_switcher.cpp b/src/boost/libs/spirit/test/lex/lexer_state_switcher.cpp
new file mode 100644
index 00000000..97e5b1eb
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/lexer_state_switcher.cpp
@@ -0,0 +1,66 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/mpl/print.hpp>
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/spirit/include/phoenix_object.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+
+///////////////////////////////////////////////////////////////////////////////
+//  Token definition
+///////////////////////////////////////////////////////////////////////////////
+template <typename Lexer>
+struct switch_state_tokens : boost::spirit::lex::lexer<Lexer>
+{
+    // define tokens and associate them with the lexer
+    switch_state_tokens()
+    {
+        namespace phoenix = boost::phoenix;
+        using boost::spirit::lex::_state;
+
+        identifier = "[a-zA-Z_][a-zA-Z0-9_]*";
+        this->self = identifier [ phoenix::ref(state_) = _state ];
+
+        integer = "[0-9]+";
+        this->self("INT") = integer [ _state = "INITIAL" ];
+    }
+
+    std::string state_;
+    boost::spirit::lex::token_def<> identifier, integer;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+int main()
+{
+    using namespace boost::spirit;
+    using namespace boost::spirit::lex;
+
+    typedef std::string::iterator base_iterator_type;
+    typedef boost::spirit::lex::lexertl::token<base_iterator_type> token_type;
+    typedef boost::spirit::lex::lexertl::actor_lexer<token_type> lexer_type;
+
+    {
+        switch_state_tokens<lexer_type> lex;
+
+        {
+            // verify whether using _state as an rvalue works
+            std::string input("abc123");
+            base_iterator_type first = input.begin();
+            BOOST_TEST(boost::spirit::lex::tokenize(first, input.end(), lex) && 
+                lex.state_ == "INITIAL");
+        }
+        {
+            // verify whether using _state as an lvalue works
+            std::string input("123abc123");
+            base_iterator_type first = input.begin();
+            BOOST_TEST(boost::spirit::lex::tokenize(first, input.end(), lex, "INT") && 
+                lex.state_ == "INITIAL");
+        }
+    }
+
+    return boost::report_errors();
+}
+
diff --git a/src/boost/libs/spirit/test/lex/lexertl1.cpp b/src/boost/libs/spirit/test/lex/lexertl1.cpp
new file mode 100644
index 00000000..edbeb26d
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/lexertl1.cpp
@@ -0,0 +1,113 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/lex_lexertl_position_token.hpp>
+#include "test.hpp"
+
+///////////////////////////////////////////////////////////////////////////////
+int main()
+{
+    using namespace boost::spirit;
+    using namespace spirit_test;
+
+    // the following test aims at the low level lexer and token_def objects, 
+    // normally not visible to/directly used by the user
+
+    // initialize tokens
+    typedef lex::token_def<std::string> token_def;
+
+    std::size_t const CCOMMENT = 1;
+    std::size_t const CPPCOMMENT = 2;
+    token_def c_comment ("\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/", CCOMMENT);
+    token_def cpp_comment ("\\/\\/[^\\n\\r]*(\\n|\\r|\\r\\n)", CPPCOMMENT);
+
+    typedef std::string::iterator base_iterator_type;
+
+    // test with default token type
+    typedef lex::lexertl::token<base_iterator_type> token_type;
+    typedef lex::lexertl::lexer<token_type> lexer_type;
+    typedef lex::lexer<lexer_type> lexer_def;
+
+    {
+        // initialize lexer
+        lexer_def lex;
+        lex.self = c_comment;
+        lex.self += cpp_comment;
+
+        // test lexer for two different input strings
+        BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));
+        BOOST_TEST(test (lex, "// this is a comment as well\n", CPPCOMMENT));
+    }
+
+    {
+        // initialize lexer
+        lexer_def lex;
+        lex.self = c_comment | cpp_comment;
+
+        // test lexer for two different input strings
+        BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));
+        BOOST_TEST(test (lex, "// this is a comment as well\n", CPPCOMMENT));
+    }
+
+    {
+        // initialize lexer
+        lexer_def lex;
+        lex.self = token_def('+') | '-' | c_comment;
+        lex.self += lex::char_('*') | '/' | cpp_comment;
+
+        // test lexer for two different input strings
+        BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));
+        BOOST_TEST(test (lex, "// this is a comment as well\n", CPPCOMMENT));
+        BOOST_TEST(test (lex, "+", '+'));
+        BOOST_TEST(test (lex, "-", '-'));
+        BOOST_TEST(test (lex, "*", '*'));
+        BOOST_TEST(test (lex, "/", '/'));
+    }
+
+    // test with position_token 
+    typedef lex::lexertl::position_token<base_iterator_type> position_token_type;
+    typedef lex::lexertl::lexer<position_token_type> position_lexer_type;
+    typedef lex::lexer<position_lexer_type> position_lexer_def;
+
+    {
+        // initialize lexer
+        position_lexer_def lex;
+        lex.self = c_comment;
+        lex.self += cpp_comment;
+
+        // test lexer for two different input strings
+        BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));
+        BOOST_TEST(test (lex, "// this is a comment as well\n", CPPCOMMENT));
+    }
+
+    {
+        // initialize lexer
+        position_lexer_def lex;
+        lex.self = c_comment | cpp_comment;
+
+        // test lexer for two different input strings
+        BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));
+        BOOST_TEST(test (lex, "// this is a comment as well\n", CPPCOMMENT));
+    }
+
+    {
+        // initialize lexer
+        position_lexer_def lex;
+        lex.self = token_def('+') | '-' | c_comment;
+        lex.self += lex::char_('*') | '/' | cpp_comment;
+
+        // test lexer for two different input strings
+        BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));
+        BOOST_TEST(test (lex, "// this is a comment as well\n", CPPCOMMENT));
+        BOOST_TEST(test (lex, "+", '+'));
+        BOOST_TEST(test (lex, "-", '-'));
+        BOOST_TEST(test (lex, "*", '*'));
+        BOOST_TEST(test (lex, "/", '/'));
+    }
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/lexertl2.cpp b/src/boost/libs/spirit/test/lex/lexertl2.cpp
new file mode 100644
index 00000000..59fafbd0
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/lexertl2.cpp
@@ -0,0 +1,91 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <string>
+
+#include "test.hpp"
+
+///////////////////////////////////////////////////////////////////////////////
+// a simple lexer class
+template <typename Lexer>
+struct lexertl_test 
+  : boost::spirit::lex::lexer<Lexer>
+{
+    typedef boost::spirit::lex::token_def<std::string> token_def;
+
+    static std::size_t const CCOMMENT = 1;
+    static std::size_t const CPPCOMMENT = 2;
+
+    lexertl_test()
+      : c_comment("\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/", CCOMMENT)
+      , cpp_comment("\\/\\/[^\\n\\r]*(\\n|\\r|\\r\\n)", CPPCOMMENT)
+    {
+        this->self = c_comment;
+        this->self += cpp_comment;
+    }
+
+    token_def c_comment, cpp_comment;
+};
+
+template <typename Lexer>
+struct wlexertl_test 
+  : boost::spirit::lex::lexer<Lexer>
+{
+    typedef boost::spirit::lex::token_def<std::basic_string<wchar_t>, wchar_t> 
+        token_def;
+
+    static std::size_t const CCOMMENT = 1;
+    static std::size_t const CPPCOMMENT = 2;
+
+    wlexertl_test()
+      : c_comment(L"\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/", CCOMMENT)
+      , cpp_comment(L"\\/\\/[^\\n\\r]*(\\n|\\r|\\r\\n)", CPPCOMMENT)
+    {
+        this->self = c_comment;
+        this->self += cpp_comment;
+    }
+
+    token_def c_comment, cpp_comment;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+int main()
+{
+    using namespace boost::spirit;
+    using namespace spirit_test;
+
+    // the following test aims at the low level lexer_ and token_ objects, 
+    // normally not visible/used by the user
+    {
+        // initialize lexer
+        typedef std::string::iterator base_iterator_type;
+        typedef lex::lexertl::token<base_iterator_type> token_type;
+        typedef lex::lexertl::lexer<token_type> lexer_type;
+        typedef lexertl_test<lexer_type> lexer_def;
+
+        // test lexer for two different input strings
+        lexer_def lex;
+        BOOST_TEST(test (lex, "/* this is a comment */", lexer_def::CCOMMENT));
+        BOOST_TEST(test (lex, "// this is a comment as well\n", lexer_def::CPPCOMMENT));
+    }
+
+    {
+        // initialize lexer
+        typedef std::basic_string<wchar_t>::iterator base_iterator_type;
+        typedef lex::lexertl::token<base_iterator_type> token_type;
+        typedef lex::lexertl::lexer<token_type> lexer_type;
+        typedef wlexertl_test<lexer_type> lexer_def;
+
+        // test lexer for two different input strings
+        lexer_def lex;
+        BOOST_TEST(test (lex, L"/* this is a comment */", lexer_def::CCOMMENT));
+        BOOST_TEST(test (lex, L"// this is a comment as well\n", lexer_def::CPPCOMMENT));
+    }
+
+    return boost::report_errors();
+}
+
diff --git a/src/boost/libs/spirit/test/lex/lexertl3.cpp b/src/boost/libs/spirit/test/lex/lexertl3.cpp
new file mode 100644
index 00000000..9ff70974
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/lexertl3.cpp
@@ -0,0 +1,68 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include "test.hpp"
+
+///////////////////////////////////////////////////////////////////////////////
+int main()
+{
+    using namespace boost::spirit;
+    using namespace boost::spirit::lex;
+    using namespace spirit_test;
+
+    // initialize tokens
+    typedef lex::token_def<std::string> token_def;
+
+    std::size_t const CCOMMENT = 1;
+    std::size_t const CPPCOMMENT = 2;
+    token_def c_comment ("\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/", CCOMMENT);
+    token_def cpp_comment ("\\/\\/[^\\n\\r]*(\\n|\\r|\\r\\n)", CPPCOMMENT);
+
+    typedef std::string::iterator base_iterator_type;    
+    typedef lex::lexertl::token<base_iterator_type> token_type;
+    typedef lex::lexertl::lexer<token_type> lexer_type;
+
+    typedef lex::lexer<lexer_type> lexer_def;
+
+    {
+        // initialize lexer
+        std::string str("def");
+        token_def ws_tok ("[\\v\\f\\n\\r]+");
+        lexer_def lex;
+        lex.self = c_comment;
+        lex.self += cpp_comment | '1' | '2' | '3' | "abc" | str;
+        lex.self += token_def(' ') | '\t' | ws_tok;
+
+        // test lexer for two different input strings
+        BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));
+        BOOST_TEST(test (lex, "// this is a comment as well\n", CPPCOMMENT));
+        BOOST_TEST(test (lex, "\n\n\v\f\r", ws_tok.id()));
+        BOOST_TEST(test (lex, " ", ' '));
+        BOOST_TEST(test (lex, "2", '2'));
+        BOOST_TEST(test (lex, "abc"));
+        BOOST_TEST(test (lex, "def"));
+    }
+
+    {
+        // initialize lexer
+        lexer_def lex;
+        token_def ws_tok ("[\\v\\f\\n\\r]+");
+        lex.self = c_comment;
+        lex.self += cpp_comment | '1' | '2' | '3';
+        lex.self("WHITESPACE") = token_def(' ') | '\t' | ws_tok;
+
+        // test lexer for two different input strings
+        BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));
+        BOOST_TEST(test (lex, "// this is a comment as well\n", CPPCOMMENT));
+        BOOST_TEST(test (lex, "2", '2'));
+        BOOST_TEST(!test (lex, "\n\n\v\f\r", ws_tok.id()));
+        BOOST_TEST(test (lex, " ", ' ', "WHITESPACE"));
+        BOOST_TEST(test (lex, "\n\n\v\f\r", ws_tok.id(), "WHITESPACE"));
+    }
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/lexertl4.cpp b/src/boost/libs/spirit/test/lex/lexertl4.cpp
new file mode 100644
index 00000000..80730528
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/lexertl4.cpp
@@ -0,0 +1,90 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include "test.hpp"
+
+///////////////////////////////////////////////////////////////////////////////
+int main()
+{
+    using namespace boost::spirit;
+    using namespace boost::spirit::lex;
+    using namespace spirit_test;
+
+    // initialize tokens
+    typedef lex::token_def<std::string> token_def;
+
+    std::size_t const CCOMMENT = 1;
+    std::size_t const CPPCOMMENT = 2;
+    std::size_t const TOKEN_ID_ABC = 1000;
+    std::size_t const TOKEN_ID_STR = 1001;
+    std::size_t const TOKEN_ID_WS = 1002;
+
+    token_def c_comment ("\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/", CCOMMENT);
+    token_def cpp_comment ("\\/\\/[^\\n\\r]*(\\n|\\r|\\r\\n)", CPPCOMMENT);
+    token_def ws_tok ("[\\v\\f\\n\\r]+", TOKEN_ID_WS);
+
+    typedef std::string::iterator base_iterator_type;    
+    typedef lex::lexertl::token<base_iterator_type> token_type;
+    typedef lex::lexertl::lexer<token_type> lexer_type;
+
+    typedef lex::lexer<lexer_type> lexer_def;
+
+    std::string str("def");
+
+    {
+        // initialize lexer
+        lexer_def lex;
+        token_def ws_tok ("[\\v\\f\\n\\r]+", TOKEN_ID_WS);
+        lex.self.add
+            (c_comment)(cpp_comment) 
+            ('1')('2')('3')
+            ("abc", TOKEN_ID_ABC)
+            (str, TOKEN_ID_STR)
+        ;
+        lex.self += token_def(' ') | '\t' | ws_tok;
+
+        // test lexer for different input strings
+        BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));
+        BOOST_TEST(test (lex, "// this is a comment as well\n", CPPCOMMENT));
+        BOOST_TEST(test (lex, "\n\n\v\f\r", ws_tok.id()));
+        BOOST_TEST(test (lex, " ", ' '));
+        BOOST_TEST(test (lex, "2", '2'));
+        BOOST_TEST(test (lex, "abc", TOKEN_ID_ABC));
+        BOOST_TEST(test (lex, "def", TOKEN_ID_STR));
+    }
+
+    {
+        // initialize lexer
+        lexer_def lex;
+        token_def ws_tok ("[\\v\\f\\n\\r]+", TOKEN_ID_WS);
+
+        lex.self.add
+            (c_comment)(cpp_comment) 
+            ('1')('2')('3')
+            ("abc", TOKEN_ID_ABC)
+            (str, TOKEN_ID_STR)
+        ;
+
+        lex.self("WHITESPACE").add
+            (' ')('\t')
+            (ws_tok)
+        ;
+
+        // test lexer for different input strings
+        BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));
+        BOOST_TEST(test (lex, "// this is a comment as well\n", CPPCOMMENT));
+        BOOST_TEST(test (lex, "2", '2'));
+        BOOST_TEST(test (lex, "abc", TOKEN_ID_ABC));
+        BOOST_TEST(test (lex, "def", TOKEN_ID_STR));
+
+        BOOST_TEST(!test (lex, "\n\n\v\f\r", TOKEN_ID_WS));
+        BOOST_TEST(test (lex, " ", ' ', "WHITESPACE"));
+        BOOST_TEST(test (lex, "\n\n\v\f\r", TOKEN_ID_WS, "WHITESPACE"));
+    }
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/lexertl5.cpp b/src/boost/libs/spirit/test/lex/lexertl5.cpp
new file mode 100644
index 00000000..33c3b3cd
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/lexertl5.cpp
@@ -0,0 +1,107 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include "test.hpp"
+
+///////////////////////////////////////////////////////////////////////////////
+//  test pattern definition capabilities
+int main()
+{
+    using namespace boost::spirit;
+    using namespace boost::spirit::lex;
+    using namespace spirit_test;
+
+    // initialize tokens
+    typedef lex::token_def<std::string> token_def;
+
+    std::size_t const CCOMMENT = 1;
+    std::size_t const CPPCOMMENT = 2;
+    std::size_t const TOKEN_ID_ABC = 1000;
+    std::size_t const TOKEN_ID_STR = 1001;
+    std::size_t const TOKEN_ID_WS = 1002;
+
+    typedef std::string::iterator base_iterator_type;    
+    typedef lex::lexertl::token<base_iterator_type> token_type;
+    typedef lex::lexertl::lexer<token_type> lexer_type;
+
+    typedef lex::lexer<lexer_type> lexer_def;
+
+    std::string str("def");
+
+    {
+        // initialize lexer
+        lexer_def lex;
+
+        lex.self.add_pattern
+            ("CCOMMENT", "\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/")
+            ("CPPCOMMENT", "\\/\\/[^\\n\\r]*(\\n|\\r|\\r\\n)")
+            ("WS", "[\\v\\f\\n\\r]+")
+        ;
+
+        token_def c_comment ("{CCOMMENT}", CCOMMENT);
+        token_def cpp_comment ("{CPPCOMMENT}", CPPCOMMENT);
+        token_def ws_tok ("{WS}");
+
+        lex.self.add
+            (c_comment)(cpp_comment) 
+            ('1')('2')('3')
+            ("abc", TOKEN_ID_ABC)
+            (str, TOKEN_ID_STR)
+        ;
+        lex.self += token_def(' ') | '\t' | ws_tok;
+
+        // test lexer for different input strings
+        BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));
+        BOOST_TEST(test (lex, "// this is a comment as well\n", CPPCOMMENT));
+        BOOST_TEST(test (lex, "\n\n\v\f\r", ws_tok.id()));
+        BOOST_TEST(test (lex, " ", ' '));
+        BOOST_TEST(test (lex, "2", '2'));
+        BOOST_TEST(test (lex, "abc", TOKEN_ID_ABC));
+        BOOST_TEST(test (lex, "def", TOKEN_ID_STR));
+    }
+
+    {
+        // initialize lexer
+        lexer_def lex;
+
+        lex.self.add_pattern
+            ("CCOMMENT", "\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/")
+            ("CPPCOMMENT", "\\/\\/[^\\n\\r]*(\\n|\\r|\\r\\n)")
+            ("WS", "[\\v\\f\\n\\r]+")
+        ;
+
+        token_def c_comment ("{CCOMMENT}", CCOMMENT);
+        token_def cpp_comment ("{CPPCOMMENT}", CPPCOMMENT);
+        token_def ws_tok ("{WS}");
+
+        // init lexer
+        lex.self.add
+            (c_comment)(cpp_comment) 
+            ('1')('2')('3')
+            ("abc", TOKEN_ID_ABC)
+            (str, TOKEN_ID_STR)
+        ;
+
+        lex.self("WHITESPACE").add
+            (' ')('\t')
+            (ws_tok, TOKEN_ID_WS)
+        ;
+
+        // test lexer for different input strings
+        BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));
+        BOOST_TEST(test (lex, "// this is a comment as well\n", CPPCOMMENT));
+        BOOST_TEST(test (lex, "2", '2'));
+        BOOST_TEST(test (lex, "abc", TOKEN_ID_ABC));
+        BOOST_TEST(test (lex, "def", TOKEN_ID_STR));
+
+        BOOST_TEST(!test (lex, "\n\n\v\f\r", TOKEN_ID_WS));
+        BOOST_TEST(test (lex, " ", ' ', "WHITESPACE"));
+        BOOST_TEST(test (lex, "\n\n\v\f\r", TOKEN_ID_WS, "WHITESPACE"));
+    }
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/matlib.h b/src/boost/libs/spirit/test/lex/matlib.h
new file mode 100644
index 00000000..defa4332
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/matlib.h
@@ -0,0 +1,100 @@
+//  Copyright (c) 2001-2009 Hartmut Kaiser
+//  Copyright (c) 2009 Carl Barron
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#ifndef MATLIB_H_05102009
+#define MATLIB_H_05102009
+#include <boost/spirit/include/lex.hpp>
+#include <vector>
+#include <string>
+
+struct set_lexer_state
+{
+    std::string state;
+    set_lexer_state(const std::string &a):state(a){}
+    template <class Iterator,class Context>
+    void operator () (Iterator const&, Iterator const&
+      , BOOST_SCOPED_ENUM(boost::spirit::lex::pass_flags)&, std::size_t
+      , Context &ctx) const
+    {
+        ctx.set_state_name(state.c_str());
+    }
+};
+
+struct store_double
+{
+    std::vector<double> &out;
+    store_double(std::vector<double> &a):out(a){}
+    template <class Iterator,class LexerContext>
+    void operator () (Iterator const& start, Iterator const& end
+      , BOOST_SCOPED_ENUM(boost::spirit::lex::pass_flags)&, std::size_t
+      , LexerContext &) const
+    {
+        std::string work(start, end);
+        out.push_back(std::atof(work.c_str()));
+    }
+
+    // silence MSVC warning C4512: assignment operator could not be generated
+    BOOST_DELETED_FUNCTION(store_double& operator= (store_double const&));
+};
+
+struct add_row
+{
+    std::vector<std::vector<double> > &matrix;
+    std::vector<double> &row;
+
+    add_row(std::vector<std::vector<double> > &a,std::vector<double> &b)
+        :matrix(a),row(b) {}
+    template <class Iterator,class Context>
+    void operator () (Iterator const&, Iterator const&
+      , BOOST_SCOPED_ENUM(boost::spirit::lex::pass_flags)&, std::size_t
+      , Context &ctx) const
+    {
+        matrix.push_back(std::vector<double>());
+        matrix.back().swap(row);
+        ctx.set_state_name("A");
+    }
+
+    // silence MSVC warning C4512: assignment operator could not be generated
+    BOOST_DELETED_FUNCTION(add_row& operator= (add_row const&));
+};
+
+template <class Lexer>
+struct matlib_tokens : boost::spirit::lex::lexer<Lexer>
+{
+    matlib_tokens(std::vector<std::vector<double> > &a)
+      : matrix(a)
+    {
+        typedef boost::spirit::lex::token_def<> token_def_;
+
+        this->self.add_pattern("REAL1", "[0-9]+(\\.[0-9]*)?");
+        this->self.add_pattern("REAL2", "\\.[0-9]+");
+
+        number = "[-+]?({REAL1}|{REAL2})([eE][-+]?[0-9]+)?";
+
+        this->self 
+            =   token_def_('[') [set_lexer_state("A")]
+            ;
+
+        this->self("A") 
+            =   token_def_('[') [set_lexer_state("B")]
+            |   ','
+            |   token_def_(']') [set_lexer_state("INITIAL")]
+            ;
+
+        this->self("B") 
+            =   number [store_double(row)]
+            |   ','
+            |   token_def_(']') [add_row(matrix,row)]
+            ;
+    }
+
+    boost::spirit::lex::token_def<> number;
+    std::vector<std::vector<double> > &matrix;
+    std::vector<double> row;
+};
+
+#endif
+
diff --git a/src/boost/libs/spirit/test/lex/pch.hpp b/src/boost/libs/spirit/test/lex/pch.hpp
new file mode 100644
index 00000000..ba5c72dd
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/pch.hpp
@@ -0,0 +1,17 @@
+/*=============================================================================
+    Copyright (c) 2019 Nikita Kniazev
+
+    Distributed under the Boost Software License, Version 1.0. (See accompanying
+    file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+=============================================================================*/
+#ifdef BOOST_BUILD_PCH_ENABLED
+
+#include <boost/spirit/include/lex.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/phoenix_core.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+#include <boost/core/lightweight_test.hpp>
+#include <string>
+#include <iostream>
+
+#endif
diff --git a/src/boost/libs/spirit/test/lex/plain_token.cpp b/src/boost/libs/spirit/test/lex/plain_token.cpp
new file mode 100644
index 00000000..4ca79b83
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/plain_token.cpp
@@ -0,0 +1,91 @@
+//  Copyright (c) 2001-2010 Hartmut Kaiser
+//  Copyright (c) 2016 Jeffrey E. Trull
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/config/warning_disable.hpp>
+
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/qi_parse.hpp>
+#include <boost/spirit/include/qi_operator.hpp>
+
+#include <string>
+
+namespace qi = boost::spirit::qi;
+namespace lex = boost::spirit::lex;
+
+enum tokenids
+{
+    // left tokens
+    IDLPAREN = lex::min_token_id, 
+    IDLANGLE,
+    IDLBRACE,
+    IDLSQUARE,
+    // right tokens
+    IDRPAREN,
+    IDRANGLE,
+    IDRBRACE,
+    IDRSQUARE,
+    IDANY
+};
+
+template <typename Lexer>
+struct delimiter_tokens : lex::lexer<Lexer>
+{
+    delimiter_tokens()
+    {
+        this->self = 
+                lex::char_('(', IDLPAREN)
+            |   lex::char_(')', IDRPAREN)
+            |   lex::char_('<', IDLANGLE)
+            |   lex::char_('>', IDRANGLE)
+            |   lex::char_('{', IDLBRACE)
+            |   lex::char_('}', IDRBRACE)
+            |   lex::char_('[', IDLSQUARE)
+            |   lex::char_(']', IDRSQUARE)
+            |   lex::string(".", IDANY)
+            ;
+    }
+};
+
+int main()
+{
+    typedef lex::lexertl::token<
+        std::string::iterator, boost::mpl::vector<std::string>
+    > token_type;
+
+    typedef lex::lexertl::lexer<token_type> lexer_type;
+    delimiter_tokens<lexer_type> delims;
+
+    // two test cases for the token range
+    std::string angled_delimiter_str("<canvas>");
+
+    using qi::token;
+    // angle brackets
+    std::string::iterator beg = angled_delimiter_str.begin();
+    BOOST_TEST(lex::tokenize_and_parse(
+                   beg, angled_delimiter_str.end(),
+                   delims,
+                   token(IDLPAREN, IDLSQUARE)
+                     >> +token(IDANY)
+                     >> token(IDRPAREN, IDRSQUARE)));
+                                       
+    std::string paren_delimiter_str("(setq foo nil)");
+    beg = paren_delimiter_str.begin();
+    BOOST_TEST(lex::tokenize_and_parse(
+                   beg, paren_delimiter_str.end(),
+                   delims,
+                   token(IDLPAREN, IDLSQUARE)
+                     >> +token(IDANY)
+                     >> token(IDRPAREN, IDRSQUARE)));
+
+    // reset and use a regular plain token
+    beg = paren_delimiter_str.begin();
+    BOOST_TEST(lex::tokenize_and_parse(
+                   beg, paren_delimiter_str.end(),
+                   delims,
+                   token(IDLPAREN) >> +token(IDANY) >> token(IDRPAREN)));
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/regression_basic_lexer.cpp b/src/boost/libs/spirit/test/lex/regression_basic_lexer.cpp
new file mode 100644
index 00000000..214e08c2
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/regression_basic_lexer.cpp
@@ -0,0 +1,64 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+//  Copyright (c) 2009 Pavel Baranov
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+
+#include <iostream>
+#include <string>
+
+using namespace boost::spirit;
+using namespace boost::spirit::lex;
+
+typedef const char * base_iterator;
+
+///////////////////////////////////////////////////////////////////////////////
+//  Token definition
+///////////////////////////////////////////////////////////////////////////////
+template <typename Lexer>
+struct position_helper_tokens : lexer<Lexer> 
+{
+    position_helper_tokens()
+    {
+        // define tokens and associate them with the lexer
+        eol = "\n";
+        any = "[^\n]+";
+
+        // associate tokens with the lexer
+        this->self
+            =   eol
+            |   any
+            ;
+    }
+
+    token_def<> any, eol;
+};
+
+int main()
+{
+    // read input from the given file
+    std::string str ("test");
+
+    // token type
+    typedef lexertl::token<base_iterator, lex::omit, boost::mpl::false_> token_type;
+
+    // lexer type
+    typedef lexertl::actor_lexer<token_type> lexer_type;
+
+    // create the lexer object instance needed to invoke the lexical analysis
+    position_helper_tokens<lexer_type> position_helper_lexer;
+
+    // tokenize the given string, all generated tokens are discarded
+    base_iterator first = str.c_str(); 
+    base_iterator last = &first[str.size()];
+
+    for(lexer_type::iterator_type i = position_helper_lexer.begin(first, last); 
+        i != position_helper_lexer.end() && (*i).is_valid(); i++ )
+    {
+    }
+    return boost::report_errors();
+}
+
diff --git a/src/boost/libs/spirit/test/lex/regression_file_iterator1.cpp b/src/boost/libs/spirit/test/lex/regression_file_iterator1.cpp
new file mode 100644
index 00000000..677f9c04
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/regression_file_iterator1.cpp
@@ -0,0 +1,99 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+//  Copyright (c) 2010 Mathias Gaunard
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/config/warning_disable.hpp>
+#include <boost/detail/lightweight_test.hpp>
+
+#include <boost/spirit/include/support_multi_pass.hpp>
+#include <boost/spirit/include/classic_position_iterator.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+
+namespace spirit = boost::spirit;
+namespace lex = spirit::lex;
+
+typedef spirit::classic::position_iterator2<
+    spirit::multi_pass<std::istreambuf_iterator<char> >
+> file_iterator;
+
+typedef boost::iterator_range<file_iterator> file_range;
+
+inline file_iterator 
+make_file_iterator(std::istream& input, const std::string& filename)
+{
+    return file_iterator(
+        spirit::make_default_multi_pass(
+            std::istreambuf_iterator<char>(input)),
+        spirit::multi_pass<std::istreambuf_iterator<char> >(),
+        filename);
+}
+
+struct identifier
+{
+    identifier(file_iterator, file_iterator)
+    {
+    }
+};
+
+struct string_literal
+{
+    string_literal(file_iterator, file_iterator)
+    {
+    }
+};
+
+typedef lex::lexertl::token<
+    file_iterator, boost::mpl::vector<identifier, string_literal>
+> token_type;
+
+struct lexer
+  : lex::lexer<lex::lexertl::actor_lexer<token_type> >
+{
+    lexer() 
+      : id("[a-zA-Z0-9]+", 1)
+      , st("'[^'\\n]*'", 2)
+    {
+        self("ST") = 
+                st [ lex::_state = "INITIAL" ] 
+            ;
+        
+        self("*") =  
+                id                       [ lex::_state = "ST" ]
+            |   lex::token_def<>(".", 3) [ lex::_state = "ST" ]
+            ;
+    }
+    
+    lex::token_def<identifier> id;
+    lex::token_def<string_literal> st;
+};
+
+typedef lexer::iterator_type token_iterator;
+
+int main()
+{
+    std::stringstream ss;
+    ss << "foo 'bar'";
+    
+    file_iterator begin = make_file_iterator(ss, "SS");
+    file_iterator end;
+    
+    lexer l;
+    token_iterator begin2 = l.begin(begin, end, "ST");
+    token_iterator end2 = l.end();
+    
+    std::size_t test_data[] = { 1, 3, 2 };
+    std::size_t const test_data_size = sizeof(test_data)/sizeof(test_data[0]);
+
+    token_iterator it = begin2;
+    std::size_t i = 0;
+    for (/**/; it != end2 && i < test_data_size; ++it, ++i)
+    {
+        BOOST_TEST(it->id() == test_data[i]);
+    }
+    BOOST_TEST(it == end2);
+    BOOST_TEST(i == test_data_size);
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/regression_file_iterator2.cpp b/src/boost/libs/spirit/test/lex/regression_file_iterator2.cpp
new file mode 100644
index 00000000..aca7f5ce
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/regression_file_iterator2.cpp
@@ -0,0 +1,106 @@
+//  Copyright (c) 2001-2010 Hartmut Kaiser
+//  Copyright (c) 2010 Mathias Gaunard
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/config/warning_disable.hpp>
+#include <boost/detail/lightweight_test.hpp>
+
+#include <boost/spirit/include/support_multi_pass.hpp>
+#include <boost/spirit/include/classic_position_iterator.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+
+namespace spirit = boost::spirit;
+namespace lex = spirit::lex;
+
+typedef spirit::classic::position_iterator2<
+    spirit::multi_pass<std::istreambuf_iterator<char> >
+> file_iterator;
+
+inline file_iterator 
+make_file_iterator(std::istream& input, const std::string& filename)
+{
+    return file_iterator(
+        spirit::make_default_multi_pass(
+            std::istreambuf_iterator<char>(input)),
+        spirit::multi_pass<std::istreambuf_iterator<char> >(),
+        filename);
+}
+
+struct identifier
+{
+    identifier(file_iterator, file_iterator)
+    {
+    }
+};
+
+struct string_literal
+{
+    string_literal(file_iterator, file_iterator)
+    {
+    }
+};
+
+typedef lex::lexertl::token<
+    file_iterator, boost::mpl::vector<identifier, string_literal>
+> token_type;
+
+struct lexer
+  : lex::lexer<lex::lexertl::actor_lexer<token_type> >
+{
+    lexer() 
+      : id("[a-zA-Z0-9]+", 1)
+      , st("'[^'\\n]*'", 2)
+    {
+        self =  id [ 
+                    lex::_state = "ST"
+                ]
+            |   lex::token_def<>(".", 3) [ 
+                    lex::_state = "ST" 
+                ]
+            ;
+        
+        self("ST") =
+                st [ 
+                    lex::_state = "INITIAL" 
+                ]
+            |   lex::token_def<>(".", 4) [ 
+                    lex::_state = "INITIAL"
+                  , lex::_pass = lex::pass_flags::pass_fail 
+                ]
+            ;
+    }
+    
+    lex::token_def<identifier> id;
+    lex::token_def<string_literal> st;
+};
+
+typedef lexer::iterator_type token_iterator;
+
+int main()
+{
+    std::stringstream ss;
+    ss << "foo 'bar'";
+    
+    file_iterator begin = make_file_iterator(ss, "SS");
+    file_iterator end;
+    
+    lexer l;
+    token_iterator begin2 = l.begin(begin, end, "ST");
+    token_iterator end2 = l.end();
+    
+    std::size_t test_data[] = { 1, 3, 2 };
+    std::size_t const test_data_size = sizeof(test_data)/sizeof(test_data[0]);
+
+    token_iterator it = begin2;
+    std::size_t i = 0;
+    for (/**/; it != end2 && i < test_data_size; ++it, ++i)
+    {
+        BOOST_TEST(it->id() == test_data[i]);
+    }
+    BOOST_TEST(it == end2);
+    BOOST_TEST(i == test_data_size);
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/regression_file_iterator3.cpp b/src/boost/libs/spirit/test/lex/regression_file_iterator3.cpp
new file mode 100644
index 00000000..cb38cf3f
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/regression_file_iterator3.cpp
@@ -0,0 +1,102 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+//  Copyright (c) 2010 Mathias Gaunard
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#define BOOST_SPIRIT_DEBUG 1    // required for token streaming
+// #define BOOST_SPIRIT_LEXERTL_DEBUG 1
+
+#include <boost/config/warning_disable.hpp>
+#include <boost/detail/lightweight_test.hpp>
+
+#include <boost/spirit/include/support_multi_pass.hpp>
+#include <boost/spirit/include/classic_position_iterator.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+
+#include <boost/spirit/include/phoenix_core.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+#include <boost/spirit/include/phoenix_statement.hpp>
+
+#include <sstream>
+
+namespace spirit = boost::spirit;
+namespace lex = spirit::lex;
+namespace phoenix = boost::phoenix;
+
+typedef spirit::classic::position_iterator2<
+    spirit::multi_pass<std::istreambuf_iterator<char> >
+> file_iterator;
+
+typedef boost::iterator_range<file_iterator> file_range;
+
+inline file_iterator 
+make_file_iterator(std::istream& input, const std::string& filename)
+{
+    return file_iterator(
+        spirit::make_default_multi_pass(
+            std::istreambuf_iterator<char>(input)),
+        spirit::multi_pass<std::istreambuf_iterator<char> >(),
+        filename);
+}
+
+struct string_literal
+{
+    string_literal(file_iterator, file_iterator)
+    {
+    }
+};
+
+typedef lex::lexertl::token<
+    file_iterator, boost::mpl::vector<string_literal>
+> token_type;
+
+struct lexer
+  : lex::lexer<lex::lexertl::actor_lexer<token_type> >
+{
+    lexer() : st("'[^'\\n]*'", 1)
+    {
+        lex::token_def<> string_lookahead('\'');
+        self("LA") = string_lookahead;
+
+        // make sure lookahead is implicitly evaluated using the lexer state
+        // the token_def has been associated with
+        self = st [
+                phoenix::if_(lex::lookahead(string_lookahead)) [ lex::more() ]
+            ]
+            ;
+    }
+    
+    lex::token_def<string_literal> st;
+};
+
+typedef lexer::iterator_type token_iterator;
+
+int main()
+{
+    std::stringstream ss;
+    ss << "'foo''bar'";
+
+    file_iterator begin = make_file_iterator(ss, "SS");
+    file_iterator end;
+
+    lexer l;
+    token_iterator begin2 = l.begin(begin, end);
+    token_iterator end2 = l.end();
+
+    char const* test_data[] = { "1,'foo'", "1,'foo''bar'" };
+    std::size_t const test_data_size = sizeof(test_data)/sizeof(test_data[0]);
+
+    token_iterator it = begin2;
+    std::size_t i = 0;
+    for (/**/; it != end2 && i < test_data_size; ++it, ++i)
+    {
+        std::stringstream ss;
+        ss << it->id() << "," << *it;
+        BOOST_TEST(ss.str() == test_data[i]);
+    }
+    BOOST_TEST(it == end2);
+    BOOST_TEST(i == test_data_size);
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/regression_file_iterator4.cpp b/src/boost/libs/spirit/test/lex/regression_file_iterator4.cpp
new file mode 100644
index 00000000..6212c203
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/regression_file_iterator4.cpp
@@ -0,0 +1,92 @@
+//  Copyright (c) 2001-2010 Hartmut Kaiser
+//  Copyright (c) 2010 Mathias Gaunard
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+// This test makes sure that the BOL state (begin of line) is properly reset
+// if a token matched at the beginning of a line is discarded using 
+// lex::pass_fail.
+
+#include <boost/config/warning_disable.hpp>
+#include <boost/detail/lightweight_test.hpp>
+
+#include <boost/spirit/include/support_multi_pass.hpp>
+#include <boost/spirit/include/classic_position_iterator.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+
+namespace spirit = boost::spirit;
+namespace lex = spirit::lex;
+
+typedef spirit::classic::position_iterator2<
+    spirit::multi_pass<std::istreambuf_iterator<char> >
+> file_iterator;
+
+inline file_iterator 
+make_file_iterator(std::istream& input, const std::string& filename)
+{
+    return file_iterator(
+        spirit::make_default_multi_pass(
+            std::istreambuf_iterator<char>(input)),
+        spirit::multi_pass<std::istreambuf_iterator<char> >(),
+        filename);
+}
+
+typedef lex::lexertl::token<file_iterator> token_type;
+
+struct lexer
+  : lex::lexer<lex::lexertl::actor_lexer<token_type> >
+{
+    lexer() : word("^[a-zA-Z0-9]+$", 1)
+    {
+        self =  word [ 
+                    lex::_state = "O" 
+                ]
+            |   lex::token_def<>("!.*$") [ 
+                    lex::_state = "O"
+                  , lex::_pass = lex::pass_flags::pass_ignore 
+                ]
+            |   lex::token_def<>('\n', 2) [ 
+                    lex::_state = "O" 
+                ] 
+            ;
+        
+        self("O") = 
+                lex::token_def<>(".") [ 
+                    lex::_state = "INITIAL"
+                  , lex::_pass = lex::pass_flags::pass_fail 
+                ]
+            ;
+    }
+    
+    lex::token_def<> word;
+};
+
+typedef lexer::iterator_type token_iterator;
+
+int main()
+{
+    std::stringstream ss;
+    ss << "!foo\nbar\n!baz";
+    
+    file_iterator begin = make_file_iterator(ss, "SS");
+    file_iterator end;
+    
+    lexer l;
+    token_iterator begin2 = l.begin(begin, end);
+    token_iterator end2 = l.end();
+    
+    std::size_t test_data[] = { 2, 1, 2 };
+    std::size_t const test_data_size = sizeof(test_data)/sizeof(test_data[0]);
+
+    token_iterator it = begin2;
+    std::size_t i = 0;
+    for (/**/; it != end2 && i < test_data_size; ++it, ++i)
+    {
+        BOOST_TEST(it->id() == test_data[i]);
+    }
+    BOOST_TEST(it == end2);
+    BOOST_TEST(i == test_data_size);
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/regression_less_8563.cpp b/src/boost/libs/spirit/test/lex/regression_less_8563.cpp
new file mode 100644
index 00000000..85858ca6
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/regression_less_8563.cpp
@@ -0,0 +1,39 @@
+//  Copyright (c) 2013 Andreas Pokorny
+//
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/config/warning_disable.hpp>
+
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/phoenix_core.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+
+#include <fstream>
+
+using namespace std;
+using namespace boost::spirit;
+
+template <typename BaseLexer>
+struct test_lexer : boost::spirit::lex::lexer<BaseLexer>
+{
+    test_lexer()
+    {
+        this->self = lex::string("just something")
+            [
+                lex::_end = lex::less(boost::phoenix::val(1))
+            ]
+            ;
+    }
+};
+
+int main()
+{
+    typedef lex::lexertl::token<char const*> token_type;
+    typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+    test_lexer<lexer_type> lexer;
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/regression_matlib_dynamic.cpp b/src/boost/libs/spirit/test/lex/regression_matlib_dynamic.cpp
new file mode 100644
index 00000000..cbeab1f5
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/regression_matlib_dynamic.cpp
@@ -0,0 +1,44 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+//  Copyright (c) 2009 Carl Barron
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <iostream>
+#include <algorithm>
+#include "matlib.h"
+
+void test_matrix(std::vector<std::vector<double> > const& x)
+{
+    BOOST_TEST(x.size() == 3);
+    BOOST_TEST(x[0].size() == 2 && x[0][0] == 1 && x[0][1] == 2);
+    BOOST_TEST(x[1].size() == 1 && x[1][0] == 3);
+    BOOST_TEST(x[2].size() == 3 && x[2][0] == 4 && x[2][1] == 5 && x[2][2] == 6);
+}
+
+int main ()
+{
+    std::string input("[[1,2][3][4,5,6]]");
+
+    std::vector<std::vector<double> > results;
+    typedef std::string::iterator iter;
+    typedef boost::spirit::lex::lexertl::actor_lexer<
+        boost::spirit::lex::lexertl::token<iter> 
+    > lexer_type;
+
+    typedef matlib_tokens<lexer_type> matlib_type;
+    matlib_type  matrix(results);
+    iter first = input.begin();
+
+    try {
+        BOOST_TEST(boost::spirit::lex::tokenize(first, input.end(), matrix));
+        test_matrix(results);
+    }
+    catch (std::runtime_error const& e) {
+        std::cerr << "caught exception: " << e.what() << std::endl;
+        BOOST_TEST(false);
+    }
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/regression_matlib_generate.cpp b/src/boost/libs/spirit/test/lex/regression_matlib_generate.cpp
new file mode 100644
index 00000000..b152b9ae
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/regression_matlib_generate.cpp
@@ -0,0 +1,32 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+//  Copyright (c) 2009 Carl Barron
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/lex_generate_static_lexertl.hpp>
+
+#include <fstream>
+#include <vector>
+
+#include "matlib.h"
+
+int main(int argc, char* argv[])
+{
+    std::vector<std::vector<double> > results;
+
+    typedef std::string::iterator iter;
+    typedef boost::spirit::lex::lexertl::actor_lexer<
+        boost::spirit::lex::lexertl::token<iter>
+    > lexer_type;
+
+    typedef matlib_tokens<lexer_type> matlib_type;
+    matlib_type matrix(results);
+
+    std::ofstream out(argc < 2 ? "matlib_static.h" : argv[1]);
+    BOOST_TEST(boost::spirit::lex::lexertl::generate_static_dfa(matrix, out, "matlib"));
+    return boost::report_errors();
+}
+
diff --git a/src/boost/libs/spirit/test/lex/regression_matlib_generate_switch.cpp b/src/boost/libs/spirit/test/lex/regression_matlib_generate_switch.cpp
new file mode 100644
index 00000000..c5003759
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/regression_matlib_generate_switch.cpp
@@ -0,0 +1,33 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+//  Copyright (c) 2009 Carl Barron
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/lex_generate_static_lexertl.hpp>
+
+#include <fstream>
+#include <vector>
+
+#include "matlib.h"
+
+int main(int argc, char* argv[])
+{
+    std::vector<std::vector<double> > results;
+
+    typedef std::string::iterator iter;
+    typedef boost::spirit::lex::lexertl::actor_lexer<
+        boost::spirit::lex::lexertl::token<iter>
+    > lexer_type;
+
+    typedef matlib_tokens<lexer_type> matlib_type;
+    matlib_type matrix(results);
+
+    std::ofstream out(argc < 2 ? "matlib_static_switch.h" : argv[1]);
+    BOOST_TEST(boost::spirit::lex::lexertl::generate_static_switch(
+        matrix, out, "matlib_switch"));
+    return boost::report_errors();
+}
+
diff --git a/src/boost/libs/spirit/test/lex/regression_matlib_static.cpp b/src/boost/libs/spirit/test/lex/regression_matlib_static.cpp
new file mode 100644
index 00000000..9512888e
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/regression_matlib_static.cpp
@@ -0,0 +1,50 @@
+//  Copyright (c) 2001-2010 Hartmut Kaiser
+//  Copyright (c) 2009 Carl Barron
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/spirit/include/lex_static_lexertl.hpp>
+
+#include <iostream>
+#include <string>
+#include <vector>
+#include <exception>
+
+#include "matlib_static.h"
+#include "matlib.h"
+
+void test_matrix(std::vector<std::vector<double> > const& x)
+{
+    BOOST_TEST(x.size() == 3);
+    BOOST_TEST(x[0].size() == 2 && x[0][0] == 1 && x[0][1] == 2);
+    BOOST_TEST(x[1].size() == 1 && x[1][0] == 3);
+    BOOST_TEST(x[2].size() == 3 && x[2][0] == 4 && x[2][1] == 5 && x[2][2] == 6);
+}
+
+int main()
+{
+    std::string input("[[1,2][3][4,5,6]]");
+    std::vector<std::vector<double> > results;
+
+    typedef std::string::iterator iter;
+    typedef boost::spirit::lex::lexertl::static_actor_lexer<
+        boost::spirit::lex::lexertl::token<iter>,
+        boost::spirit::lex::lexertl::static_::lexer_matlib
+    > lexer_type;
+
+    typedef matlib_tokens<lexer_type> matlib_type;
+    matlib_type matrix(results);
+    iter first = input.begin();
+
+    try {
+        BOOST_TEST(boost::spirit::lex::tokenize(first, input.end(), matrix));
+        test_matrix(results);
+    }
+    catch (std::runtime_error const& e) {
+        std::cerr << e.what() << '\n';
+        BOOST_TEST(false);
+    }
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/regression_matlib_switch.cpp b/src/boost/libs/spirit/test/lex/regression_matlib_switch.cpp
new file mode 100644
index 00000000..dc8e315b
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/regression_matlib_switch.cpp
@@ -0,0 +1,50 @@
+//  Copyright (c) 2001-2010 Hartmut Kaiser
+//  Copyright (c) 2009 Carl Barron
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/spirit/include/lex_static_lexertl.hpp>
+
+#include <iostream>
+#include <string>
+#include <vector>
+#include <exception>
+
+#include "matlib_static_switch.h"
+#include "matlib.h"
+
+void test_matrix(std::vector<std::vector<double> > const& x)
+{
+    BOOST_TEST(x.size() == 3);
+    BOOST_TEST(x[0].size() == 2 && x[0][0] == 1 && x[0][1] == 2);
+    BOOST_TEST(x[1].size() == 1 && x[1][0] == 3);
+    BOOST_TEST(x[2].size() == 3 && x[2][0] == 4 && x[2][1] == 5 && x[2][2] == 6);
+}
+
+int main()
+{
+    std::string input("[[1,2][3][4,5,6]]");
+    std::vector<std::vector<double> > results;
+
+    typedef std::string::iterator iter;
+    typedef boost::spirit::lex::lexertl::static_actor_lexer<
+        boost::spirit::lex::lexertl::token<iter>,
+        boost::spirit::lex::lexertl::static_::lexer_matlib_switch
+    > lexer_type;
+
+    typedef matlib_tokens<lexer_type> matlib_type;
+    matlib_type matrix(results);
+    iter first = input.begin();
+
+    try {
+        BOOST_TEST(boost::spirit::lex::tokenize(first, input.end(), matrix));
+        test_matrix(results);
+    }
+    catch (std::runtime_error const& e) {
+        std::cerr << e.what() << '\n';
+        BOOST_TEST(false);
+    }
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/regression_static_wide_6253.cpp b/src/boost/libs/spirit/test/lex/regression_static_wide_6253.cpp
new file mode 100644
index 00000000..816fe98c
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/regression_static_wide_6253.cpp
@@ -0,0 +1,44 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+//  Copyright (c) 2011 Ryan Molden
+//
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/config/warning_disable.hpp>
+
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/lex_generate_static_lexertl.hpp>
+#include <boost/spirit/include/lex_static_lexertl.hpp>
+
+#include <fstream>
+
+using namespace std;
+using namespace boost::spirit;
+
+template <typename BaseLexer>
+struct my_lexer : boost::spirit::lex::lexer<BaseLexer>
+{
+    my_lexer()
+    {
+        token = L"Yay winning!";
+        this->self = token;
+    }
+
+    lex::token_def<lex::unused_type, wchar_t> token;
+};
+
+int main()
+{
+    typedef lex::lexertl::token<wchar_t const*> token_type;
+    typedef lex::lexertl::lexer<token_type> lexer_type;
+
+    my_lexer<lexer_type> lexer;
+
+    basic_ofstream<wchar_t> output_dfa("test_dfa.hpp");
+    BOOST_TEST(lex::lexertl::generate_static_dfa(lexer, output_dfa, L"test_dfa"));
+
+    basic_ofstream<wchar_t> output_switch("test_switch.hpp");
+    BOOST_TEST(lex::lexertl::generate_static_switch(lexer, output_switch, L"test_switch"));
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/regression_syntax_error.cpp b/src/boost/libs/spirit/test/lex/regression_syntax_error.cpp
new file mode 100644
index 00000000..eb3decba
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/regression_syntax_error.cpp
@@ -0,0 +1,96 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+//  Copyright (c) 2009 Jean-Francois Ostiguy
+//
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/config/warning_disable.hpp>
+
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/qi_parse.hpp>
+#include <boost/spirit/include/qi_operator.hpp>
+#include <boost/spirit/include/qi_char.hpp>
+#include <boost/spirit/include/qi_grammar.hpp>
+#include <boost/spirit/include/qi_eoi.hpp>
+
+#include <string>
+#include <iostream>
+#include <sstream>
+
+namespace lex = boost::spirit::lex;
+namespace qi = boost::spirit::qi;
+namespace mpl = boost::mpl;
+
+template <typename Lexer>
+struct my_lexer : lex::lexer<Lexer>
+{
+    my_lexer() 
+    {
+        delimiter = "BEGIN|END";
+        identifier = "[a-zA-Z][_\\.a-zA-Z0-9]*";
+        ws = "[ \\t\\n]+";
+        real = "([0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?)|([-+]?[1-9]+\\.?([eE][-+]?[0-9]+))";
+        integer = "[0-9]+"; 
+
+        this->self += ws[lex::_pass = lex::pass_flags::pass_ignore];
+        this->self += delimiter;
+        this->self += identifier;
+        this->self += real;
+        this->self += integer;
+        this->self += '=';
+        this->self += ';';
+    }
+
+    lex::token_def<> ws;
+    lex::token_def<std::string> identifier;
+    lex::token_def<int> integer;
+    lex::token_def<double> real;
+    lex::token_def<> delimiter;
+};
+
+template <typename Iterator>
+struct my_grammar : qi::grammar<Iterator> 
+{
+    template <typename TokenDef>
+    my_grammar( TokenDef const& tok )
+      :  my_grammar::base_type(statement) 
+    {
+        statement 
+            =   qi::eoi
+            |  *(delimiter | declaration)
+            ;
+
+        delimiter = tok.delimiter >> tok.identifier;
+        declaration = tok.identifier >> option >> ';';
+        option = *(tok.identifier >> '=' >> (tok.real | tok.integer));
+    }
+
+    qi::rule<Iterator> statement, delimiter, declaration, option;
+};
+
+typedef lex::lexertl::token<char const*
+  , mpl::vector<std::string, double, int> > token_type;
+typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+typedef my_lexer<lexer_type>::iterator_type iterator_type;
+
+int main()
+{
+    std::string test_string ("BEGIN section\n");
+    // we introduce a syntax error: ";;" instead of ";" as a terminator.
+    test_string += "Identity;;\n";      // this will make the parser fail 
+    test_string += "END section\n" ;
+
+    char const* first = &test_string[0];
+    char const* last  = &first[test_string.size()];
+
+    my_lexer<lexer_type> lexer;
+    my_grammar<iterator_type> grammar(lexer);
+
+    BOOST_TEST(lex::tokenize_and_parse(first, last, lexer, grammar));
+    BOOST_TEST(first != last);
+
+    return boost::report_errors();
+}
+
+
diff --git a/src/boost/libs/spirit/test/lex/regression_wide.cpp b/src/boost/libs/spirit/test/lex/regression_wide.cpp
new file mode 100644
index 00000000..11e7c3a4
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/regression_wide.cpp
@@ -0,0 +1,131 @@
+//  Copyright (c) 2001-2010 Hartmut Kaiser
+//  Copyright (c) 2010 Sergey "GooRoo" Olendarenko
+//
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/config/warning_disable.hpp>
+
+#include <cstdlib>
+#include <iostream>
+#include <locale>
+#include <string>
+
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/phoenix_function.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+
+namespace lex = boost::spirit::lex;
+namespace phoenix = boost::phoenix;
+
+typedef std::basic_string<wchar_t> wstring_type;
+
+///////////////////////////////////////////////////////////////////////////////
+enum tokenids 
+{
+    ID_IDENT = 1,
+    ID_CONSTANT,
+    ID_OPERATION,
+    ID_BRACKET
+};
+
+///////////////////////////////////////////////////////////////////////////////
+struct test_data
+{
+    tokenids     tokenid;
+    wstring_type value;
+};
+
+// alpha+x1*(2.836-x2[i])
+test_data data[] = 
+{
+    { ID_IDENT, L"alpha" },
+    { ID_OPERATION, L"+" },
+    { ID_IDENT, L"x1" },
+    { ID_OPERATION, L"*" },
+    { ID_BRACKET, L"(" },
+    { ID_CONSTANT, L"2.836" },
+    { ID_OPERATION, L"-" },
+    { ID_IDENT, L"x2" },
+    { ID_BRACKET, L"[" },
+    { ID_IDENT, L"i" },
+    { ID_BRACKET, L"]" },
+    { ID_BRACKET, L")" }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+struct test_impl
+{
+    typedef void result_type;
+    template <typename TokenId, typename Value>
+    struct result { typedef void type; };
+
+    template <typename TokenId, typename Value>
+    void operator()(TokenId const& tokenid, Value const& val) const
+    {
+        BOOST_TEST(sequence_counter < sizeof(data)/sizeof(data[0]));
+        BOOST_TEST(data[sequence_counter].tokenid == tokenids(tokenid));
+        BOOST_TEST(0 == val.which());
+
+        typedef boost::iterator_range<wstring_type::iterator> iterator_range;
+        iterator_range r = boost::get<iterator_range>(val);
+        BOOST_TEST(data[sequence_counter].value == 
+            wstring_type(r.begin(), r.end()));
+
+        ++sequence_counter;
+    }
+
+    static std::size_t sequence_counter;
+};
+std::size_t test_impl::sequence_counter = 0;
+
+phoenix::function<test_impl> const test = test_impl();
+
+///////////////////////////////////////////////////////////////////////////////
+template <typename Lexer>
+struct mega_tokens : lex::lexer<Lexer>
+{
+    mega_tokens()
+        : identifier(L"[a-zA-Z_][a-zA-Z0-9_]*", ID_IDENT)
+        , constant  (L"[0-9]+(\\.[0-9]+)?", ID_CONSTANT)
+        , operation (L"[\\+\\-\\*/]", ID_OPERATION)
+        , bracket   (L"[\\(\\)\\[\\]]", ID_BRACKET)
+    {
+        using lex::_tokenid;
+        using lex::_val;
+
+        this->self
+            = operation  [ test(_tokenid, _val) ]
+            | identifier [ test(_tokenid, _val) ]
+            | constant   [ test(_tokenid, _val) ]
+            | bracket    [ test(_tokenid, _val) ]
+        ;
+    }
+
+    lex::token_def<wstring_type, wchar_t, tokenids> identifier;
+    lex::token_def<double, wchar_t, tokenids> constant;
+    lex::token_def<wchar_t, wchar_t, tokenids> operation;
+    lex::token_def<wchar_t, wchar_t, tokenids> bracket;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+int main()
+{
+    typedef wstring_type::iterator base_iterator;
+    typedef lex::lexertl::token<
+        base_iterator, boost::mpl::vector<wchar_t, wstring_type, double>
+      , boost::mpl::true_, tokenids
+    > token_type;
+    typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+    mega_tokens<lexer_type> mega_lexer;
+
+    wstring_type exampleStr = L"alpha+x1*(2.836-x2[i])";
+    base_iterator first = exampleStr.begin();
+
+    BOOST_TEST(lex::tokenize(first, exampleStr.end(), mega_lexer));
+    BOOST_TEST(test_impl::sequence_counter == sizeof(data)/sizeof(data[0]));
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/regression_word_count.cpp b/src/boost/libs/spirit/test/lex/regression_word_count.cpp
new file mode 100644
index 00000000..0536d5f0
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/regression_word_count.cpp
@@ -0,0 +1,89 @@
+//  Copyright (c) 2001-2010 Hartmut Kaiser
+//  Copyright (c) 2009 Tor Brede Vekterli
+//
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/config/warning_disable.hpp>
+
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/qi_parse.hpp>
+#include <boost/spirit/include/qi_operator.hpp>
+#include <boost/spirit/include/qi_action.hpp>
+#include <boost/spirit/include/qi_char.hpp>
+#include <boost/spirit/include/qi_grammar.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+
+#include <iostream>
+#include <string>
+
+namespace qi = boost::spirit::qi;
+namespace lex = boost::spirit::lex;
+
+enum tokenids
+{
+    IDANY = lex::min_token_id + 10 // Lower 8 bits is 0x0a, same as '\n'
+};
+
+template <typename Lexer>
+struct word_count_tokens : lex::lexer<Lexer>
+{
+    word_count_tokens()
+    {
+        this->self.add_pattern
+            ("TEST", "A")
+        ;
+        word = "{TEST}";
+        this->self.add
+            (word)
+            ('\n')
+            (".", IDANY)
+        ;
+    }
+    lex::token_def<std::string> word;
+};
+
+template <typename Iterator>
+struct word_count_grammar : qi::grammar<Iterator>
+{
+    template <typename TokenDef>
+    word_count_grammar(TokenDef const& tok)
+      : word_count_grammar::base_type(start)
+      , c(0), w(0), l(0)
+    {
+        using boost::phoenix::ref;
+        using qi::lit;
+        using qi::token;
+
+        start =  *(   tok.word      [++ref(w)]
+                  |   lit('\n')     [++ref(l)]
+                  |   token(IDANY)  [++ref(c)]
+                  )
+              ;
+    }
+    std::size_t c, w, l;
+    qi::rule<Iterator> start;
+};
+
+
+int main()
+{
+    typedef lex::lexertl::token<
+        const char*, boost::mpl::vector<std::string>
+    > token_type;
+
+    typedef lex::lexertl::lexer<token_type> lexer_type;
+    typedef word_count_tokens<lexer_type>::iterator_type iterator_type;
+    word_count_tokens<lexer_type> word_count;          // Our lexer
+    word_count_grammar<iterator_type> g (word_count);  // Our parser
+
+    std::string str ("A\nBCDEFGHI");
+    char const* first = str.c_str();
+    char const* last = &first[str.size()];
+
+    BOOST_TEST(lex::tokenize_and_parse(first, last, word_count, g));
+    BOOST_TEST(g.l == 1 && g.w == 1 && g.c == 8);
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/semantic_actions.cpp b/src/boost/libs/spirit/test/lex/semantic_actions.cpp
new file mode 100644
index 00000000..129bc075
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/semantic_actions.cpp
@@ -0,0 +1,300 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+//
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/config/warning_disable.hpp>
+
+#include <boost/spirit/include/lex_lexertl.hpp>
+
+namespace lex = boost::spirit::lex;
+
+typedef lex::lexertl::token<std::string::iterator> token_type;
+typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+///////////////////////////////////////////////////////////////////////////////
+static bool found_identifier_flag = false;
+
+///////////////////////////////////////////////////////////////////////////////
+void found_identifier_sa0()
+{
+    found_identifier_flag = true;
+}
+
+template <typename Lexer>
+struct lexer_sa0 : lex::lexer<Lexer>
+{
+    lexer_sa0() 
+    {
+        identifier = "[a-zA-Z][_a-zA-Z0-9]*";
+        this->self += identifier [&found_identifier_sa0];
+    }
+    lex::token_def<> identifier;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+static std::string found_identifier_str;
+
+void found_identifier_sa2(std::string::iterator& start
+  , std::string::iterator& end)
+{
+    found_identifier_flag = true;
+    found_identifier_str = std::string(start, end);
+}
+
+template <typename Lexer>
+struct lexer_sa2 : lex::lexer<Lexer>
+{
+    lexer_sa2() 
+    {
+        identifier = "[a-zA-Z][_a-zA-Z0-9]*";
+        this->self += identifier [&found_identifier_sa2];
+    }
+    lex::token_def<> identifier;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+void found_identifier_sa3_normal(std::string::iterator& start
+  , std::string::iterator& end, BOOST_SCOPED_ENUM(lex::pass_flags)& pass)
+{
+    BOOST_TEST(pass == lex::pass_flags::pass_normal);
+
+    found_identifier_flag = true;
+    found_identifier_str = std::string(start, end);
+}
+
+template <typename Lexer>
+struct lexer_sa3_normal : lex::lexer<Lexer>
+{
+    lexer_sa3_normal() 
+    {
+        identifier = "[a-zA-Z][_a-zA-Z0-9]*";
+        this->self += identifier [&found_identifier_sa3_normal];
+    }
+    lex::token_def<> identifier;
+};
+
+void found_identifier_sa3_fail(std::string::iterator&, std::string::iterator&
+  , BOOST_SCOPED_ENUM(lex::pass_flags)& pass)
+{
+    pass = lex::pass_flags::pass_fail;
+}
+
+template <typename Lexer>
+struct lexer_sa3_fail : lex::lexer<Lexer>
+{
+    lexer_sa3_fail() 
+    {
+        identifier = "[a-zA-Z][_a-zA-Z0-9]*";
+        this->self += identifier [&found_identifier_sa3_fail];
+    }
+    lex::token_def<> identifier;
+};
+
+void found_identifier_sa3_ignore(std::string::iterator&, std::string::iterator&
+  , BOOST_SCOPED_ENUM(lex::pass_flags)& pass)
+{
+    pass = lex::pass_flags::pass_ignore;
+}
+
+template <typename Lexer>
+struct lexer_sa3_ignore : lex::lexer<Lexer>
+{
+    lexer_sa3_ignore() 
+    {
+        identifier = "[a-zA-Z][_a-zA-Z0-9]*";
+        this->self += identifier [&found_identifier_sa3_ignore];
+    }
+    lex::token_def<> identifier;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+static std::size_t found_identifier_id = 0;
+
+void found_identifier_sa4(std::string::iterator& start
+  , std::string::iterator& end, BOOST_SCOPED_ENUM(lex::pass_flags)& pass
+  , std::size_t id)
+{
+    BOOST_TEST(pass == lex::pass_flags::pass_normal);
+
+    found_identifier_flag = true;
+    found_identifier_str = std::string(start, end);
+    found_identifier_id = id;
+}
+
+template <typename Lexer>
+struct lexer_sa4 : lex::lexer<Lexer>
+{
+    lexer_sa4() 
+    {
+        identifier = "[a-zA-Z][_a-zA-Z0-9]*";
+        this->self += identifier [&found_identifier_sa4];
+    }
+    lex::token_def<> identifier;
+};
+
+void found_identifier_sa4_id(std::string::iterator& start
+  , std::string::iterator& end, BOOST_SCOPED_ENUM(lex::pass_flags)& pass
+  , std::size_t& id)
+{
+    BOOST_TEST(pass == lex::pass_flags::pass_normal);
+
+    found_identifier_flag = true;
+    found_identifier_str = std::string(start, end);
+    found_identifier_id = id;
+    id = 1;
+}
+
+template <typename Lexer>
+struct lexer_sa4_id : lex::lexer<Lexer>
+{
+    lexer_sa4_id() 
+    {
+        identifier = "[a-zA-Z][_a-zA-Z0-9]*";
+        this->self += identifier [&found_identifier_sa4_id];
+    }
+    lex::token_def<> identifier;
+};
+
+static std::size_t found_identifier_id2 = 0;
+
+bool identifier_token(token_type const& t)
+{
+    found_identifier_id2 = t.id();
+    return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+struct found_identifier_sa5
+{
+    template <typename Context>
+    void operator()(std::string::iterator& /*start*/
+      , std::string::iterator& /*end*/, BOOST_SCOPED_ENUM(lex::pass_flags)& pass
+      , std::size_t& /*id*/, Context& ctx)
+    {
+        BOOST_TEST(pass == lex::pass_flags::pass_normal);
+
+        found_identifier_flag = true;
+        found_identifier_str = std::string(ctx.get_value().begin(), ctx.get_value().end());
+    }
+};
+
+template <typename Lexer>
+struct lexer_sa5 : lex::lexer<Lexer>
+{
+    lexer_sa5() 
+    {
+        identifier = "[a-zA-Z][_a-zA-Z0-9]*";
+        this->self += identifier [found_identifier_sa5()];
+    }
+    lex::token_def<> identifier;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+int main()
+{
+    std::string identifier ("id_1234");
+    std::string::iterator first = identifier.begin();
+    std::string::iterator last = identifier.end();
+
+    // test semantic action taking no arguments
+    found_identifier_flag = false;
+    {
+        lexer_sa0<lexer_type> sa0;
+        BOOST_TEST(lex::tokenize(first, last, sa0));
+        BOOST_TEST(first == last);
+        BOOST_TEST(found_identifier_flag);
+    }
+
+    // test semantic action taking two arguments (iterator pair for matched 
+    // sequence)
+    found_identifier_flag = false;
+    found_identifier_str.clear();
+    first = identifier.begin();
+    {
+        lexer_sa2<lexer_type> sa2;
+        BOOST_TEST(lex::tokenize(first, last, sa2));
+        BOOST_TEST(first == last);
+        BOOST_TEST(found_identifier_flag);
+        BOOST_TEST(found_identifier_str == identifier);
+    }
+
+    // test semantic action taking three arguments (iterator pair for matched 
+    // sequence and pass_flags) - pass_flags::pass_normal
+    found_identifier_flag = false;
+    found_identifier_str.clear();
+    first = identifier.begin();
+    {
+        lexer_sa3_normal<lexer_type> sa3;
+        BOOST_TEST(lex::tokenize(first, last, sa3));
+        BOOST_TEST(first == last);
+        BOOST_TEST(found_identifier_flag);
+        BOOST_TEST(found_identifier_str == identifier);
+    }
+
+    // test semantic action taking three arguments (iterator pair for matched 
+    // sequence and pass_flags) - pass_flags::pass_fail
+    first = identifier.begin();
+    {
+        lexer_sa3_fail<lexer_type> sa3;
+        BOOST_TEST(!lex::tokenize(first, last, sa3));
+        BOOST_TEST(first != last);
+    }
+
+    // test semantic action taking three arguments (iterator pair for matched 
+    // sequence and pass_flags) - pass_flags::pass_ignore
+    first = identifier.begin();
+    {
+        lexer_sa3_ignore<lexer_type> sa3;
+        BOOST_TEST(lex::tokenize(first, last, sa3));
+        BOOST_TEST(first == last);
+    }
+
+    // test semantic action taking four arguments (iterator pair for matched 
+    // sequence and pass_flags, and token id)
+    found_identifier_flag = false;
+    found_identifier_str.clear();
+    first = identifier.begin();
+    found_identifier_id = 0;
+    {
+        lexer_sa4<lexer_type> sa4;
+        BOOST_TEST(lex::tokenize(first, last, sa4));
+        BOOST_TEST(first == last);
+        BOOST_TEST(found_identifier_flag);
+        BOOST_TEST(found_identifier_str == identifier);
+        BOOST_TEST(found_identifier_id == lex::min_token_id);
+    }
+
+    found_identifier_flag = false;
+    found_identifier_str.clear();
+    first = identifier.begin();
+    found_identifier_id = 0;
+    found_identifier_id2 = 0;
+    {
+        lexer_sa4_id<lexer_type> sa4;
+        BOOST_TEST(lex::tokenize(first, last, sa4, identifier_token));
+        BOOST_TEST(first == last);
+        BOOST_TEST(found_identifier_flag);
+        BOOST_TEST(found_identifier_str == identifier);
+        BOOST_TEST(found_identifier_id == lex::min_token_id);
+        BOOST_TEST(found_identifier_id2 == 1);
+    }
+
+    // test semantic action taking four arguments (iterator pair for matched 
+    // sequence and pass_flags, token id, and context)
+    found_identifier_flag = false;
+    found_identifier_str.clear();
+    first = identifier.begin();
+    found_identifier_id = 0;
+    found_identifier_id2 = 0;
+    {
+        lexer_sa5<lexer_type> sa5;
+        BOOST_TEST(lex::tokenize(first, last, sa5));
+        BOOST_TEST(first == last);
+        BOOST_TEST(found_identifier_flag);
+        BOOST_TEST(found_identifier_str == identifier);
+    }
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/set_token_value.cpp b/src/boost/libs/spirit/test/lex/set_token_value.cpp
new file mode 100644
index 00000000..1cb32206
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/set_token_value.cpp
@@ -0,0 +1,255 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/spirit/include/phoenix_object.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+#include <boost/spirit/include/phoenix_container.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/foreach.hpp>
+
+using namespace boost::spirit;
+
+///////////////////////////////////////////////////////////////////////////////
+// semantic action analyzing leading whitespace 
+enum tokenids
+{
+    ID_INDENT = 1000,
+    ID_DEDENT
+};
+
+struct handle_whitespace
+{
+    handle_whitespace(std::stack<unsigned int>& indents)
+      : indents_(indents) {}
+
+    template <typename Iterator, typename IdType, typename Context>
+    void operator()(Iterator& start, Iterator& end
+      , BOOST_SCOPED_ENUM(lex::pass_flags)& pass, IdType& id
+      , Context& ctx)
+    {
+        unsigned int level = 0;
+        if (is_indent(start, end, level)) {
+            id = ID_INDENT;
+            ctx.set_value(level);
+        }
+        else if (is_dedent(start, end, level)) {
+            id = ID_DEDENT;
+            ctx.set_value(level);
+        }
+        else {
+            pass = lex::pass_flags::pass_ignore;
+        }
+    }
+
+    // Get indentation level, for now (no tabs) we just count the spaces
+    // once we allow tabs in the regex this needs to be expanded
+    template <typename Iterator>
+    unsigned int get_indent(Iterator& start, Iterator& end)
+    {
+        return static_cast<unsigned int>(std::distance(start, end));
+    }
+
+    template <typename Iterator>
+    bool is_dedent(Iterator& start, Iterator& end, unsigned int& level) 
+    {
+        unsigned int newindent = get_indent(start, end);
+        while (!indents_.empty() && newindent < indents_.top()) {
+            level++;        // dedent one more level
+            indents_.pop();
+        }
+        return level > 0;
+    }
+
+    // Handle additional indentation
+    template <typename Iterator>
+    bool is_indent(Iterator& start, Iterator& end, unsigned int& level) 
+    {
+        unsigned int newindent = get_indent(start, end);
+        if (indents_.empty() || newindent > indents_.top()) {
+            level = 1;      // indent one more level 
+            indents_.push(newindent);
+            return true;
+        }
+        return false;
+    }
+
+    std::stack<unsigned int>& indents_;
+
+    // silence MSVC warning C4512: assignment operator could not be generated
+    BOOST_DELETED_FUNCTION(handle_whitespace& operator= (handle_whitespace const&));
+};
+
+///////////////////////////////////////////////////////////////////////////////
+//  Token definition
+template <typename Lexer>
+struct set_token_value : boost::spirit::lex::lexer<Lexer>
+{
+    set_token_value()
+    {
+        using lex::_pass;
+
+        // define tokens and associate them with the lexer
+        whitespace = "^[ ]+";
+        newline = '\n';
+
+        this->self = whitespace[ handle_whitespace(indents) ];
+        this->self += newline[ _pass = lex::pass_flags::pass_ignore ];
+    }
+
+    lex::token_def<unsigned int> whitespace;
+    lex::token_def<> newline;
+    std::stack<unsigned int> indents;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+struct token_data
+{
+    int id;
+    unsigned int value;
+};
+
+template <typename Token>
+inline 
+bool test_tokens(token_data const* d, std::vector<Token> const& tokens)
+{
+    BOOST_FOREACH(Token const& t, tokens)
+    {
+        if (d->id == -1)
+            return false;           // reached end of expected data
+
+        typename Token::token_value_type const& value (t.value());
+        if (t.id() != static_cast<std::size_t>(d->id))        // token id must match
+            return false;
+        if (value.which() != 1)     // must have an integer value 
+            return false;
+        if (boost::get<unsigned int>(value) != d->value)  // value must match
+            return false;
+        ++d;
+    }
+
+    return (d->id == -1) ? true : false;
+}
+
+inline 
+bool test_indents(int *i, std::stack<unsigned int>& indents)
+{
+    while (!indents.empty())
+    {
+        if (*i == -1)
+            return false;           // reached end of expected data
+        if (indents.top() != static_cast<unsigned int>(*i))
+            return false;           // value must match
+
+        ++i;
+        indents.pop();
+    }
+
+    return (*i == -1) ? true : false;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+int main()
+{
+    namespace lex = boost::spirit::lex;
+    namespace phoenix = boost::phoenix;
+
+    typedef std::string::iterator base_iterator_type;
+    typedef boost::mpl::vector<unsigned int> token_value_types;
+    typedef lex::lexertl::token<base_iterator_type, token_value_types> token_type;
+    typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+    // test simple indent
+    {
+        set_token_value<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        std::string input("    ");
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        int i[] = { 4, -1 };
+        BOOST_TEST(test_indents(i, lexer.indents));
+
+        token_data d[] = { { ID_INDENT, 1 }, { -1, 0 } };
+        BOOST_TEST(test_tokens(d, tokens));
+    }
+
+    // test two indents
+    {
+        set_token_value<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        std::string input(
+            "    \n"
+            "        \n");
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        int i[] = { 8, 4, -1 };
+        BOOST_TEST(test_indents(i, lexer.indents));
+
+        token_data d[] = { 
+            { ID_INDENT, 1 }, { ID_INDENT, 1 }
+          , { -1, 0 } };
+        BOOST_TEST(test_tokens(d, tokens));
+    }
+
+    // test one dedent
+    {
+        set_token_value<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        std::string input(
+            "    \n"
+            "        \n"
+            "    \n");
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        int i[] = { 4, -1 };
+        BOOST_TEST(test_indents(i, lexer.indents));
+
+        token_data d[] = { 
+            { ID_INDENT, 1 }, { ID_INDENT, 1 }
+          , { ID_DEDENT, 1 }
+          , { -1, 0 } };
+        BOOST_TEST(test_tokens(d, tokens));
+    }
+
+    // test two dedents
+    {
+        set_token_value<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        std::string input(
+            "    \n"
+            "        \n"
+            "            \n"
+            "    \n");
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        int i[] = { 4, -1 };
+        BOOST_TEST(test_indents(i, lexer.indents));
+
+        token_data d[] = { 
+            { ID_INDENT, 1 }, { ID_INDENT, 1 }, { ID_INDENT, 1 }
+          , { ID_DEDENT, 2 }
+          , { -1, 0 } };
+        BOOST_TEST(test_tokens(d, tokens));
+    }
+
+    return boost::report_errors();
+}
+
diff --git a/src/boost/libs/spirit/test/lex/set_token_value_phoenix.cpp b/src/boost/libs/spirit/test/lex/set_token_value_phoenix.cpp
new file mode 100644
index 00000000..c55ba543
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/set_token_value_phoenix.cpp
@@ -0,0 +1,68 @@
+//  Copyright (c) 2009 Carl Barron
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <string>
+#include <iostream>
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/spirit/include/lex.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/phoenix_core.hpp>
+#include <boost/spirit/include/phoenix_function.hpp>
+
+namespace lex = boost::spirit::lex;
+namespace phoenix = boost::phoenix;
+
+///////////////////////////////////////////////////////////////////////////////
+struct square_impl
+{
+    template <class>
+    struct result { typedef int type; };
+
+    template <class A>
+    int operator () (const A &x) const
+    { return (x) * (x); }
+};
+
+phoenix::function<square_impl> const square = square_impl();
+
+///////////////////////////////////////////////////////////////////////////////
+template <class Lexer>
+struct test_tokens : lex::lexer<Lexer>
+{
+    test_tokens()
+    {
+        a = "a";
+        this->self = a [lex::_val = square(*lex::_start)];
+    }
+
+    lex::token_def<int> a;
+};
+
+struct catch_result
+{
+    template <class Token>
+    bool operator() (Token const& x) const
+    {
+        BOOST_TEST(x.value().which() == 1);
+        BOOST_TEST(boost::get<int>(x.value()) == 9409);  // 9409 == 'a' * 'a'
+        return true;
+    }
+};
+
+///////////////////////////////////////////////////////////////////////////////
+int main()
+{
+    typedef lex::lexertl::token<std::string::iterator
+      , boost::mpl::vector<int> > token_type;
+
+    std::string in = "a";
+    std::string::iterator first(in.begin());
+
+    test_tokens<lex::lexertl::actor_lexer<token_type> > the_lexer;
+    BOOST_TEST(lex::tokenize(first, in.end(), the_lexer, catch_result()));
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/state_switcher.cpp b/src/boost/libs/spirit/test/lex/state_switcher.cpp
new file mode 100644
index 00000000..855e5cb2
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/state_switcher.cpp
@@ -0,0 +1,91 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/qi_parse.hpp>
+#include <boost/spirit/include/qi_operator.hpp>
+#include "test_parser.hpp"
+
+///////////////////////////////////////////////////////////////////////////////
+//  Token definition
+///////////////////////////////////////////////////////////////////////////////
+template <typename Lexer>
+struct switch_state_tokens : boost::spirit::lex::lexer<Lexer>
+{
+    switch_state_tokens()
+    {
+        // define tokens and associate them with the lexer
+        identifier = "[a-zA-Z_][a-zA-Z0-9_]*";
+        this->self = identifier;
+
+        // any token definition to be used as the skip parser during parsing 
+        // has to be associated with a separate lexer state (here 'WS') 
+        white_space = "[ \\t\\n]+";
+        this->self("WS") = white_space;
+
+        separators = "[,;]";
+        this->self("SEP") = separators;
+    }
+
+    boost::spirit::lex::token_def<> identifier, white_space, separators;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+int main()
+{
+    using namespace boost::spirit;
+    using namespace boost::spirit::qi;
+    using namespace spirit_test;
+
+    typedef std::string::iterator base_iterator_type;
+    typedef boost::spirit::lex::lexertl::token<base_iterator_type> token_type;
+    typedef boost::spirit::lex::lexertl::lexer<token_type> lexer_type;
+
+    {
+        // the tokens class will be initialized inside the test_parser function
+        switch_state_tokens<lexer_type> lex;
+
+        BOOST_TEST(test_parser("ident", lex.identifier, lex));
+        BOOST_TEST(!test_parser("ident", set_state("WS") >> lex.identifier, lex));
+        BOOST_TEST(!test_parser("ident", in_state("WS")[lex.identifier], lex));
+
+        BOOST_TEST(test_parser("\t \n", set_state("WS") >> lex.white_space, lex));
+        BOOST_TEST(test_parser("\t \n", in_state("WS")[lex.white_space], lex));
+        BOOST_TEST(!test_parser("\t \n", lex.white_space, lex));
+    }
+
+    {
+        // the tokens class will be initialized inside the test_parser function
+        switch_state_tokens<lexer_type> lex;
+
+        BOOST_TEST(test_parser(",ident", lex.identifier, lex, 
+            in_state("SEP")[lex.separators]));
+        BOOST_TEST(!test_parser(";ident", set_state("WS") >> lex.identifier, 
+            lex, in_state("SEP")[lex.separators]));
+        BOOST_TEST(!test_parser(",ident", in_state("WS")[lex.identifier], 
+            lex, in_state("SEP")[lex.separators]));
+
+        BOOST_TEST(test_parser(",\t \n", set_state("WS") >> lex.white_space, 
+            lex, in_state("SEP")[lex.separators]));
+        BOOST_TEST(test_parser(";\t \n", in_state("WS")[lex.white_space], 
+            lex, in_state("SEP")[lex.separators]));
+        BOOST_TEST(!test_parser(",\t \n", lex.white_space, lex, 
+            in_state("SEP")[lex.separators]));
+    }
+
+    {
+        // the tokens class will be initialized inside the test_parser function
+        switch_state_tokens<lexer_type> lex;
+        
+        BOOST_TEST(test_parser("ident\t \n", 
+            lex.identifier >> set_state("WS") >> lex.white_space, lex));
+        BOOST_TEST(test_parser("\t \nident", 
+            in_state("WS")[lex.white_space] >> lex.identifier, lex));
+    }
+
+    return boost::report_errors();
+}
+
diff --git a/src/boost/libs/spirit/test/lex/string_token_id.cpp b/src/boost/libs/spirit/test/lex/string_token_id.cpp
new file mode 100644
index 00000000..4c9954b4
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/string_token_id.cpp
@@ -0,0 +1,87 @@
+//  Copyright (c) 2001-2010 Hartmut Kaiser
+//
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/detail/lightweight_test.hpp>
+#include <boost/config/warning_disable.hpp>
+
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/qi_parse.hpp>
+#include <boost/spirit/include/qi_operator.hpp>
+#include <boost/spirit/include/qi_action.hpp>
+#include <boost/spirit/include/qi_grammar.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+
+#include <iostream>
+#include <string>
+
+namespace qi = boost::spirit::qi;
+namespace lex = boost::spirit::lex;
+
+enum tokenids
+{
+    IDWORD = lex::min_token_id, 
+    IDCHAR,
+    IDANY
+};
+
+template <typename Lexer>
+struct word_count_tokens : lex::lexer<Lexer>
+{
+    word_count_tokens()
+    {
+        this->self.add_pattern
+            ("TEST", "A")
+        ;
+
+        this->self = 
+                lex::string("{TEST}", IDWORD) 
+            |   lex::char_('a', IDCHAR)
+            |   lex::string(".", IDANY)
+            ;
+    }
+};
+
+template <typename Iterator>
+struct word_count_grammar : qi::grammar<Iterator>
+{
+    template <typename TokenDef>
+    word_count_grammar(TokenDef const&)
+      : word_count_grammar::base_type(start)
+      , w(0), c(0), a(0)
+    {
+        using boost::phoenix::ref;
+        using qi::token;
+
+        start =  *(   token(IDWORD) [++ref(w)]
+                  |   token(IDCHAR) [++ref(c)]
+                  |   token(IDANY)  [++ref(a)]
+                  )
+              ;
+    }
+    std::size_t w, c, a;
+    qi::rule<Iterator> start;
+};
+
+
+int main()
+{
+    typedef lex::lexertl::token<
+        const char*, boost::mpl::vector<std::string>
+    > token_type;
+
+    typedef lex::lexertl::lexer<token_type> lexer_type;
+    typedef word_count_tokens<lexer_type>::iterator_type iterator_type;
+    word_count_tokens<lexer_type> word_count;          // Our lexer
+    word_count_grammar<iterator_type> g (word_count);  // Our parser
+
+    std::string str ("AaBCD");
+    char const* first = str.c_str();
+    char const* last = &first[str.size()];
+
+    BOOST_TEST(lex::tokenize_and_parse(first, last, word_count, g));
+    BOOST_TEST(g.w == 1 && g.c == 1 && g.a == 3);
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/test.hpp b/src/boost/libs/spirit/test/lex/test.hpp
new file mode 100644
index 00000000..f8c48273
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/test.hpp
@@ -0,0 +1,92 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#if !defined(BOOST_SPIRIT_LEX_TEST_MAR_23_2007_0721PM)
+#define BOOST_SPIRIT_LEX_TEST_MAR_23_2007_0721PM
+
+#include <boost/variant.hpp>
+#include <boost/range/iterator_range.hpp>
+
+namespace spirit_test
+{
+    ///////////////////////////////////////////////////////////////////////////
+    struct display_type
+    {
+        template<typename T>
+        void operator()(T const &) const
+        {
+            std::cout << typeid(T).name() << std::endl;
+        }
+
+        template<typename T>
+        static void print() 
+        {
+            std::cout << typeid(T).name() << std::endl;
+        }
+    };
+
+    ///////////////////////////////////////////////////////////////////////////
+    display_type const display = {};
+
+    ///////////////////////////////////////////////////////////////////////////
+    template <typename Iterator>
+    inline boost::iterator_range<Iterator> const& 
+    get_iterpair(boost::iterator_range<Iterator> const& itp)
+    {
+        return itp;
+    }
+
+    template <typename Iterator, BOOST_VARIANT_ENUM_PARAMS(typename T)>
+    inline boost::iterator_range<Iterator> const& 
+    get_iterpair(boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)> const& v)
+    {
+        return boost::get<boost::iterator_range<Iterator> >(v);
+    }
+
+    ///////////////////////////////////////////////////////////////////////////
+    template <typename Lexer, typename Char>
+    inline bool 
+    test(Lexer& lex, Char const* input, std::size_t token_id = 0, 
+        Char const* state = NULL)
+    {
+        typedef typename Lexer::iterator_type iterator_type;
+        typedef std::basic_string<Char> string_type;
+
+        string_type str(input);
+        typename string_type::iterator it = str.begin();
+
+        iterator_type first = lex.begin(it, str.end());
+        iterator_type last = lex.end();
+
+        bool r = true;
+
+        if (NULL != state) {
+            std::size_t stateid = lex.map_state(state);
+            r = r && (static_cast<unsigned>(~0) != stateid);
+            first.set_state(stateid);
+        }
+
+        r = r && lex;
+        r = r && first != last;
+
+        if (token_id != 0)
+            r = r && (*first).id() == token_id;
+        else
+            r = r && (*first).id() != 0;
+
+        using namespace boost;
+
+        typedef typename Lexer::iterator_type::base_iterator_type iterator;
+        typedef iterator_range<iterator> iterpair_type;
+        iterpair_type const& ip = get_iterpair<iterator>((*first).value());
+
+        r = r && string_type(ip.begin(), ip.end()) == str;
+        return r && first != last && ++first == last;
+    }
+}
+
+#endif
+
+
diff --git a/src/boost/libs/spirit/test/lex/test_parser.hpp b/src/boost/libs/spirit/test/lex/test_parser.hpp
new file mode 100644
index 00000000..6a23cb34
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/test_parser.hpp
@@ -0,0 +1,59 @@
+/*=============================================================================
+    Copyright (c) 2001-2011 Hartmut Kaiser
+
+    Distributed under the Boost Software License, Version 1.0. (See accompanying
+    file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+=============================================================================*/
+#if !defined(BOOST_SPIRIT_TEST_PARSER_SEP_24_2007_0558PM)
+#define BOOST_SPIRIT_TEST_PARSER_SEP_24_2007_0558PM
+
+#include <boost/spirit/include/qi_parse.hpp>
+#include <boost/spirit/include/qi_what.hpp>
+
+namespace spirit_test
+{
+    template <typename Char, typename Parser, typename Lexer>
+    inline bool test_parser(Char const* in, Parser const& p, Lexer& lex, 
+        bool full_match = true)
+    {
+        // we don't care about the result of the "what" function.
+        // we only care that all parsers have it:
+        boost::spirit::qi::what(p);
+
+        std::string str (in);
+        std::string::iterator it_in = str.begin();
+        std::string::iterator end_in = str.end();
+
+        typedef typename Lexer::iterator_type iterator_type;
+
+        iterator_type iter = lex.begin(it_in, end_in);
+        iterator_type end = lex.end();
+
+        return boost::spirit::qi::parse(iter, end, p)
+            && (!full_match || (iter == end));
+    }
+
+    template <typename Char, typename Parser, typename Lexer, typename Skipper>
+    inline bool test_parser(Char const* in, Parser const& p, Lexer& lex,
+          Skipper const& s, bool full_match = true)
+    {
+        // we don't care about the result of the "what" function.
+        // we only care that all parsers have it:
+        boost::spirit::qi::what(p);
+
+        std::string str (in);
+        std::string::iterator it_in = str.begin();
+        std::string::iterator end_in = str.end();
+
+        typedef typename Lexer::iterator_type iterator_type;
+
+        iterator_type iter = lex.begin(it_in, end_in);
+        iterator_type end = lex.end();
+
+        return boost::spirit::qi::phrase_parse(iter, end, p, s)
+            && (!full_match || (iter == end));
+    }
+
+}
+
+#endif
diff --git a/src/boost/libs/spirit/test/lex/token_iterpair.cpp b/src/boost/libs/spirit/test/lex/token_iterpair.cpp
new file mode 100644
index 00000000..4e05b8e5
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/token_iterpair.cpp
@@ -0,0 +1,251 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/config/warning_disable.hpp>
+#include <boost/detail/lightweight_test.hpp>
+
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/lex_lexertl_position_token.hpp>
+#include <boost/spirit/include/phoenix_object.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+#include <boost/spirit/include/phoenix_container.hpp>
+
+namespace lex = boost::spirit::lex;
+namespace phoenix = boost::phoenix;
+namespace mpl = boost::mpl;
+
+///////////////////////////////////////////////////////////////////////////////
+enum tokenids
+{
+    ID_INT = 1000,
+    ID_DOUBLE
+};
+
+template <typename Lexer>
+struct token_definitions : lex::lexer<Lexer>
+{
+    token_definitions()
+    {
+        this->self.add_pattern("HEXDIGIT", "[0-9a-fA-F]");
+        this->self.add_pattern("OCTALDIGIT", "[0-7]");
+        this->self.add_pattern("DIGIT", "[0-9]");
+
+        this->self.add_pattern("OPTSIGN", "[-+]?");
+        this->self.add_pattern("EXPSTART", "[eE][-+]");
+        this->self.add_pattern("EXPONENT", "[eE]{OPTSIGN}{DIGIT}+");
+
+        // define tokens and associate them with the lexer
+        int_ = "(0x|0X){HEXDIGIT}+|0{OCTALDIGIT}*|{OPTSIGN}[1-9]{DIGIT}*";
+        int_.id(ID_INT);
+
+        double_ = "{OPTSIGN}({DIGIT}*\\.{DIGIT}+|{DIGIT}+\\.){EXPONENT}?|{DIGIT}+{EXPONENT}";
+        double_.id(ID_DOUBLE);
+
+        whitespace = "[ \t\n]+";
+
+        this->self = 
+                double_ 
+            |   int_ 
+            |   whitespace[ lex::_pass = lex::pass_flags::pass_ignore ]
+            ;
+    }
+
+    lex::token_def<lex::omit> int_;
+    lex::token_def<lex::omit> double_;
+    lex::token_def<lex::omit> whitespace;
+};
+
+template <typename Lexer>
+struct token_definitions_with_state : lex::lexer<Lexer>
+{
+    token_definitions_with_state()
+    {
+        this->self.add_pattern("HEXDIGIT", "[0-9a-fA-F]");
+        this->self.add_pattern("OCTALDIGIT", "[0-7]");
+        this->self.add_pattern("DIGIT", "[0-9]");
+
+        this->self.add_pattern("OPTSIGN", "[-+]?");
+        this->self.add_pattern("EXPSTART", "[eE][-+]");
+        this->self.add_pattern("EXPONENT", "[eE]{OPTSIGN}{DIGIT}+");
+
+        this->self.add_state();
+        this->self.add_state("INT");
+        this->self.add_state("DOUBLE");
+
+        // define tokens and associate them with the lexer
+        int_ = "(0x|0X){HEXDIGIT}+|0{OCTALDIGIT}*|{OPTSIGN}[1-9]{DIGIT}*";
+        int_.id(ID_INT);
+
+        double_ = "{OPTSIGN}({DIGIT}*\\.{DIGIT}+|{DIGIT}+\\.){EXPONENT}?|{DIGIT}+{EXPONENT}";
+        double_.id(ID_DOUBLE);
+
+        whitespace = "[ \t\n]+";
+
+        this->self("*") = 
+                double_ [ lex::_state = "DOUBLE"] 
+            |   int_ [ lex::_state = "INT" ]
+            |   whitespace[ lex::_pass = lex::pass_flags::pass_ignore ]
+            ;
+    }
+
+    lex::token_def<lex::omit> int_;
+    lex::token_def<lex::omit> double_;
+    lex::token_def<lex::omit> whitespace;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+template <typename Token>
+inline bool 
+test_token_ids(int const* ids, std::vector<Token> const& tokens)
+{
+    BOOST_FOREACH(Token const& t, tokens)
+    {
+        if (*ids == -1)
+            return false;           // reached end of expected data
+
+        if (t.id() != static_cast<std::size_t>(*ids))        // token id must match
+            return false;
+        ++ids;
+    }
+
+    return (*ids == -1) ? true : false;
+}
+
+template <typename Token>
+inline bool 
+test_token_states(std::size_t const* states, std::vector<Token> const& tokens)
+{
+    BOOST_FOREACH(Token const& t, tokens)
+    {
+        if (*states == std::size_t(-1))
+            return false;           // reached end of expected data
+
+        if (t.state() != *states)            // token state must match
+            return false;
+        ++states;
+    }
+
+    return (*states == std::size_t(-1)) ? true : false;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+struct position_type
+{
+    std::size_t begin, end;
+};
+
+template <typename Iterator, typename Token>
+inline bool 
+test_token_positions(Iterator begin, position_type const* positions, 
+    std::vector<Token> const& tokens)
+{
+    BOOST_FOREACH(Token const& t, tokens)
+    {
+        if (positions->begin == std::size_t(-1) && 
+            positions->end == std::size_t(-1))
+        {
+            return false;           // reached end of expected data
+        }
+
+        boost::iterator_range<Iterator> matched = t.matched();
+        std::size_t start = std::distance(begin, matched.begin());
+        std::size_t end = std::distance(begin, matched.end());
+
+        // position must match
+        if (start != positions->begin || end != positions->end)
+            return false;
+
+        ++positions;
+    }
+
+    return (positions->begin == std::size_t(-1) && 
+            positions->end == std::size_t(-1)) ? true : false;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+int main()
+{
+    typedef std::string::iterator base_iterator_type;
+    std::string input(" 01 1.2 -2 0x3 2.3e6 -3.4");
+    int ids[] = { ID_INT, ID_DOUBLE, ID_INT, ID_INT, ID_DOUBLE, ID_DOUBLE, -1 };
+    std::size_t states[] = { 0, 1, 2, 1, 1, 2, std::size_t(-1) };
+    position_type positions[] = 
+    {
+        { 1, 3 }, { 4, 7 }, { 8, 10 }, { 11, 14 }, { 15, 20 }, { 21, 25 }, 
+        { std::size_t(-1), std::size_t(-1) }
+    };
+
+    // token type: token id, iterator_pair as token value, no state
+    {
+        typedef lex::lexertl::token<
+            base_iterator_type, mpl::vector<>, mpl::false_> token_type;
+        typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+        token_definitions<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        BOOST_TEST(test_token_ids(ids, tokens));
+    }
+
+    {
+        typedef lex::lexertl::position_token<
+            base_iterator_type, mpl::vector<>, mpl::false_> token_type;
+        typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+        token_definitions<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        BOOST_TEST(test_token_ids(ids, tokens));
+        BOOST_TEST(test_token_positions(input.begin(), positions, tokens));
+    }
+
+    // token type: holds token id, state, iterator_pair as token value
+    {
+        typedef lex::lexertl::token<
+            base_iterator_type, mpl::vector<>, mpl::true_> token_type;
+        typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+        token_definitions_with_state<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        BOOST_TEST(test_token_ids(ids, tokens));
+        BOOST_TEST(test_token_states(states, tokens));
+    }
+
+    {
+        typedef lex::lexertl::position_token<
+            base_iterator_type, mpl::vector<>, mpl::true_> token_type;
+        typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+        token_definitions_with_state<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        BOOST_TEST(test_token_ids(ids, tokens));
+        BOOST_TEST(test_token_states(states, tokens));
+        BOOST_TEST(test_token_positions(input.begin(), positions, tokens));
+    }
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/token_moretypes.cpp b/src/boost/libs/spirit/test/lex/token_moretypes.cpp
new file mode 100644
index 00000000..7f50527d
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/token_moretypes.cpp
@@ -0,0 +1,298 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/config/warning_disable.hpp>
+#include <boost/detail/lightweight_test.hpp>
+
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/lex_lexertl_position_token.hpp>
+#include <boost/spirit/include/phoenix_object.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+#include <boost/spirit/include/phoenix_container.hpp>
+#include <boost/spirit/include/qi_numeric.hpp>
+
+namespace spirit = boost::spirit;
+namespace lex = boost::spirit::lex;
+namespace phoenix = boost::phoenix;
+namespace mpl = boost::mpl;
+
+///////////////////////////////////////////////////////////////////////////////
+enum tokenids
+{
+    ID_INT = 1000,
+    ID_DOUBLE
+};
+
+template <typename Lexer>
+struct token_definitions : lex::lexer<Lexer>
+{
+    token_definitions()
+    {
+        this->self.add_pattern("HEXDIGIT", "[0-9a-fA-F]");
+        this->self.add_pattern("OCTALDIGIT", "[0-7]");
+        this->self.add_pattern("DIGIT", "[0-9]");
+
+        this->self.add_pattern("OPTSIGN", "[-+]?");
+        this->self.add_pattern("EXPSTART", "[eE][-+]");
+        this->self.add_pattern("EXPONENT", "[eE]{OPTSIGN}{DIGIT}+");
+
+        // define tokens and associate them with the lexer
+        int_ = "(0x|0X){HEXDIGIT}+|0{OCTALDIGIT}*|{OPTSIGN}[1-9]{DIGIT}*";
+        int_.id(ID_INT);
+
+        double_ = "{OPTSIGN}({DIGIT}*\\.{DIGIT}+|{DIGIT}+\\.){EXPONENT}?|{DIGIT}+{EXPONENT}";
+        double_.id(ID_DOUBLE);
+
+        whitespace = "[ \t\n]+";
+
+        this->self = 
+                double_ 
+            |   int_ 
+            |   whitespace[ lex::_pass = lex::pass_flags::pass_ignore ]
+            ;
+    }
+
+    lex::token_def<int> int_;
+    lex::token_def<double> double_;
+    lex::token_def<lex::omit> whitespace;
+};
+
+template <typename Lexer>
+struct token_definitions_with_state : lex::lexer<Lexer>
+{
+    token_definitions_with_state()
+    {
+        this->self.add_pattern("HEXDIGIT", "[0-9a-fA-F]");
+        this->self.add_pattern("OCTALDIGIT", "[0-7]");
+        this->self.add_pattern("DIGIT", "[0-9]");
+
+        this->self.add_pattern("OPTSIGN", "[-+]?");
+        this->self.add_pattern("EXPSTART", "[eE][-+]");
+        this->self.add_pattern("EXPONENT", "[eE]{OPTSIGN}{DIGIT}+");
+
+        this->self.add_state();
+        this->self.add_state("INT");
+        this->self.add_state("DOUBLE");
+
+        // define tokens and associate them with the lexer
+        int_ = "(0x|0X){HEXDIGIT}+|0{OCTALDIGIT}*|{OPTSIGN}[1-9]{DIGIT}*";
+        int_.id(ID_INT);
+
+        double_ = "{OPTSIGN}({DIGIT}*\\.{DIGIT}+|{DIGIT}+\\.){EXPONENT}?|{DIGIT}+{EXPONENT}";
+        double_.id(ID_DOUBLE);
+
+        whitespace = "[ \t\n]+";
+
+        this->self("*") = 
+                double_ [ lex::_state = "DOUBLE"] 
+            |   int_ [ lex::_state = "INT" ]
+            |   whitespace[ lex::_pass = lex::pass_flags::pass_ignore ]
+            ;
+    }
+
+    lex::token_def<int> int_;
+    lex::token_def<double> double_;
+    lex::token_def<lex::omit> whitespace;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+template <typename Token>
+inline bool 
+test_token_ids(int const* ids, std::vector<Token> const& tokens)
+{
+    BOOST_FOREACH(Token const& t, tokens)
+    {
+        if (*ids == -1)
+            return false;           // reached end of expected data
+
+        if (t.id() != static_cast<std::size_t>(*ids))        // token id must match
+            return false;
+
+        ++ids;
+    }
+
+    return (*ids == -1) ? true : false;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+template <typename Token>
+inline bool 
+test_token_states(std::size_t const* states, std::vector<Token> const& tokens)
+{
+    BOOST_FOREACH(Token const& t, tokens)
+    {
+        if (*states == std::size_t(-1))
+            return false;           // reached end of expected data
+
+        if (t.state() != *states)            // token state must match
+            return false;
+
+        ++states;
+    }
+
+    return (*states == std::size_t(-1)) ? true : false;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+struct position_type
+{
+    std::size_t begin, end;
+};
+
+template <typename Iterator, typename Token>
+inline bool 
+test_token_positions(Iterator begin, position_type const* positions, 
+    std::vector<Token> const& tokens)
+{
+    BOOST_FOREACH(Token const& t, tokens)
+    {
+        if (positions->begin == std::size_t(-1) && 
+            positions->end == std::size_t(-1))
+        {
+            return false;           // reached end of expected data
+        }
+
+        boost::iterator_range<Iterator> matched = t.matched();
+        std::size_t start = std::distance(begin, matched.begin());
+        std::size_t end = std::distance(begin, matched.end());
+
+        // position must match
+        if (start != positions->begin || end != positions->end)
+            return false;
+
+        ++positions;
+    }
+
+    return (positions->begin == std::size_t(-1) && 
+            positions->end == std::size_t(-1)) ? true : false;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+template <typename T, typename Token>
+inline bool 
+test_token_values(boost::optional<T> const* values, std::vector<Token> const& tokens)
+{
+    BOOST_FOREACH(Token const& t, tokens)
+    {
+        if (values->is_initialized() && values->get() == 0)
+            return false;               // reached end of expected data
+
+        if (values->is_initialized()) {
+            T val;
+            spirit::traits::assign_to(t, val);
+            if (val != values->get())   // token value must match
+                return false;
+        }
+
+        ++values;
+    }
+
+    return (values->is_initialized() && values->get() == 0) ? true : false;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+int main()
+{
+    using boost::none;
+    typedef std::string::iterator base_iterator_type;
+    std::string input(" 01 1.2 -2 03  2.3e6 -3.4");
+    int ids[] = { ID_INT, ID_DOUBLE, ID_INT, ID_INT, ID_DOUBLE, ID_DOUBLE, -1 };
+    std::size_t states[] = { 0, 1, 2, 1, 1, 2, std::size_t(-1) };
+    position_type positions[] = 
+    {
+        { 1, 3 }, { 4, 7 }, { 8, 10 }, { 11, 13 }, { 15, 20 }, { 21, 25 }, 
+        { std::size_t(-1), std::size_t(-1) }
+    };
+    boost::optional<int> ivalues[] = { 
+        1, none, -2, 
+        3, none, none, 
+        0
+    };
+    boost::optional<double> dvalues[] = { 
+        none, 1.2, none, 
+        none, 2.3e6, -3.4, 
+        0.0
+    };
+
+    // token type: token id, iterator_pair as token value, no state
+    {
+        typedef lex::lexertl::token<
+            base_iterator_type, mpl::vector<double, int>, mpl::false_> token_type;
+        typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+        token_definitions<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        BOOST_TEST(test_token_ids(ids, tokens));
+        BOOST_TEST(test_token_values(ivalues, tokens));
+        BOOST_TEST(test_token_values(dvalues, tokens));
+    }
+
+    {
+        typedef lex::lexertl::position_token<
+            base_iterator_type, mpl::vector<double, int>, mpl::false_> token_type;
+        typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+        token_definitions<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        BOOST_TEST(test_token_ids(ids, tokens));
+        BOOST_TEST(test_token_positions(input.begin(), positions, tokens));
+        BOOST_TEST(test_token_values(ivalues, tokens));
+        BOOST_TEST(test_token_values(dvalues, tokens));
+    }
+
+    // token type: holds token id, state, iterator_pair as token value
+    {
+        typedef lex::lexertl::token<
+            base_iterator_type, mpl::vector<double, int>, mpl::true_> token_type;
+        typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+        token_definitions_with_state<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        BOOST_TEST(test_token_ids(ids, tokens));
+        BOOST_TEST(test_token_states(states, tokens));
+        BOOST_TEST(test_token_values(ivalues, tokens));
+        BOOST_TEST(test_token_values(dvalues, tokens));
+    }
+
+    {
+        typedef lex::lexertl::position_token<
+            base_iterator_type, mpl::vector<double, int>, mpl::true_> token_type;
+        typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+        token_definitions_with_state<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        BOOST_TEST(test_token_ids(ids, tokens));
+        BOOST_TEST(test_token_states(states, tokens));
+        BOOST_TEST(test_token_positions(input.begin(), positions, tokens));
+        BOOST_TEST(test_token_values(ivalues, tokens));
+        BOOST_TEST(test_token_values(dvalues, tokens));
+    }
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/token_omit.cpp b/src/boost/libs/spirit/test/lex/token_omit.cpp
new file mode 100644
index 00000000..9b38591a
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/token_omit.cpp
@@ -0,0 +1,254 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/config/warning_disable.hpp>
+#include <boost/detail/lightweight_test.hpp>
+
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/lex_lexertl_position_token.hpp>
+#include <boost/spirit/include/phoenix_object.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+#include <boost/spirit/include/phoenix_container.hpp>
+
+namespace lex = boost::spirit::lex;
+namespace phoenix = boost::phoenix;
+namespace mpl = boost::mpl;
+
+///////////////////////////////////////////////////////////////////////////////
+enum tokenids
+{
+    ID_INT = 1000,
+    ID_DOUBLE
+};
+
+template <typename Lexer>
+struct token_definitions : lex::lexer<Lexer>
+{
+    token_definitions()
+    {
+        this->self.add_pattern("HEXDIGIT", "[0-9a-fA-F]");
+        this->self.add_pattern("OCTALDIGIT", "[0-7]");
+        this->self.add_pattern("DIGIT", "[0-9]");
+
+        this->self.add_pattern("OPTSIGN", "[-+]?");
+        this->self.add_pattern("EXPSTART", "[eE][-+]");
+        this->self.add_pattern("EXPONENT", "[eE]{OPTSIGN}{DIGIT}+");
+
+        // define tokens and associate them with the lexer
+        int_ = "(0x|0X){HEXDIGIT}+|0{OCTALDIGIT}*|{OPTSIGN}[1-9]{DIGIT}*";
+        int_.id(ID_INT);
+
+        double_ = "{OPTSIGN}({DIGIT}*\\.{DIGIT}+|{DIGIT}+\\.){EXPONENT}?|{DIGIT}+{EXPONENT}";
+        double_.id(ID_DOUBLE);
+
+        whitespace = "[ \t\n]+";
+
+        this->self = 
+                double_ 
+            |   int_ 
+            |   whitespace[ lex::_pass = lex::pass_flags::pass_ignore ]
+            ;
+    }
+
+    lex::token_def<lex::omit> int_;
+    lex::token_def<lex::omit> double_;
+    lex::token_def<lex::omit> whitespace;
+};
+
+template <typename Lexer>
+struct token_definitions_with_state : lex::lexer<Lexer>
+{
+    token_definitions_with_state()
+    {
+        this->self.add_pattern("HEXDIGIT", "[0-9a-fA-F]");
+        this->self.add_pattern("OCTALDIGIT", "[0-7]");
+        this->self.add_pattern("DIGIT", "[0-9]");
+
+        this->self.add_pattern("OPTSIGN", "[-+]?");
+        this->self.add_pattern("EXPSTART", "[eE][-+]");
+        this->self.add_pattern("EXPONENT", "[eE]{OPTSIGN}{DIGIT}+");
+
+        this->self.add_state();
+        this->self.add_state("INT");
+        this->self.add_state("DOUBLE");
+
+        // define tokens and associate them with the lexer
+        int_ = "(0x|0X){HEXDIGIT}+|0{OCTALDIGIT}*|{OPTSIGN}[1-9]{DIGIT}*";
+        int_.id(ID_INT);
+
+        double_ = "{OPTSIGN}({DIGIT}*\\.{DIGIT}+|{DIGIT}+\\.){EXPONENT}?|{DIGIT}+{EXPONENT}";
+        double_.id(ID_DOUBLE);
+
+        whitespace = "[ \t\n]+";
+
+        this->self("*") = 
+                double_ [ lex::_state = "DOUBLE"] 
+            |   int_ [ lex::_state = "INT" ]
+            |   whitespace[ lex::_pass = lex::pass_flags::pass_ignore ]
+            ;
+    }
+
+    lex::token_def<lex::omit> int_;
+    lex::token_def<lex::omit> double_;
+    lex::token_def<lex::omit> whitespace;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+template <typename Token>
+inline bool 
+test_token_ids(int const* ids, std::vector<Token> const& tokens)
+{
+    BOOST_FOREACH(Token const& t, tokens)
+    {
+        if (*ids == -1)
+            return false;           // reached end of expected data
+
+        if (t.id() != static_cast<std::size_t>(*ids))        // token id must match
+            return false;
+
+        ++ids;
+    }
+
+    return (*ids == -1) ? true : false;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+template <typename Token>
+inline bool 
+test_token_states(std::size_t const* states, std::vector<Token> const& tokens)
+{
+    BOOST_FOREACH(Token const& t, tokens)
+    {
+        if (*states == std::size_t(-1))
+            return false;           // reached end of expected data
+
+        if (t.state() != *states)            // token state must match
+            return false;
+
+        ++states;
+    }
+
+    return (*states == std::size_t(-1)) ? true : false;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+struct position_type
+{
+    std::size_t begin, end;
+};
+
+template <typename Iterator, typename Token>
+inline bool 
+test_token_positions(Iterator begin, position_type const* positions, 
+    std::vector<Token> const& tokens)
+{
+    BOOST_FOREACH(Token const& t, tokens)
+    {
+        if (positions->begin == std::size_t(-1) && 
+            positions->end == std::size_t(-1))
+        {
+            return false;           // reached end of expected data
+        }
+
+        boost::iterator_range<Iterator> matched = t.matched();
+        std::size_t start = std::distance(begin, matched.begin());
+        std::size_t end = std::distance(begin, matched.end());
+
+        // position must match
+        if (start != positions->begin || end != positions->end)
+            return false;
+
+        ++positions;
+    }
+
+    return (positions->begin == std::size_t(-1) && 
+            positions->end == std::size_t(-1)) ? true : false;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+int main()
+{
+    typedef std::string::iterator base_iterator_type;
+    std::string input(" 01 1.2 -2 0x3 2.3e6 -3.4");
+    int ids[] = { ID_INT, ID_DOUBLE, ID_INT, ID_INT, ID_DOUBLE, ID_DOUBLE, -1 };
+    std::size_t states[] = { 0, 1, 2, 1, 1, 2, std::size_t(-1) };
+    position_type positions[] = 
+    {
+        { 1, 3 }, { 4, 7 }, { 8, 10 }, { 11, 14 }, { 15, 20 }, { 21, 25 }, 
+        { std::size_t(-1), std::size_t(-1) }
+    };
+
+    // minimal token type: holds just token id, no state, no value
+    {
+        typedef lex::lexertl::token<
+            base_iterator_type, lex::omit, mpl::false_> token_type;
+        typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+        token_definitions<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        BOOST_TEST(test_token_ids(ids, tokens));
+    }
+
+    {
+        typedef lex::lexertl::position_token<
+            base_iterator_type, lex::omit, mpl::false_> token_type;
+        typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+        token_definitions<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        BOOST_TEST(test_token_ids(ids, tokens));
+        BOOST_TEST(test_token_positions(input.begin(), positions, tokens));
+    }
+
+    // minimal token type: holds just token id and state, no value
+    {
+        typedef lex::lexertl::token<
+            base_iterator_type, lex::omit, mpl::true_> token_type;
+        typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+        token_definitions_with_state<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        BOOST_TEST(test_token_ids(ids, tokens));
+        BOOST_TEST(test_token_states(states, tokens));
+    }
+
+    {
+        typedef lex::lexertl::position_token<
+            base_iterator_type, lex::omit, mpl::true_> token_type;
+        typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+        token_definitions_with_state<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        BOOST_TEST(test_token_ids(ids, tokens));
+        BOOST_TEST(test_token_states(states, tokens));
+        BOOST_TEST(test_token_positions(input.begin(), positions, tokens));
+    }
+
+    return boost::report_errors();
+}
diff --git a/src/boost/libs/spirit/test/lex/token_onetype.cpp b/src/boost/libs/spirit/test/lex/token_onetype.cpp
new file mode 100644
index 00000000..62f0f18b
--- /dev/null
+++ b/src/boost/libs/spirit/test/lex/token_onetype.cpp
@@ -0,0 +1,282 @@
+//  Copyright (c) 2001-2011 Hartmut Kaiser
+// 
+//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
+//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+#include <boost/config/warning_disable.hpp>
+#include <boost/detail/lightweight_test.hpp>
+
+#include <boost/spirit/include/lex_lexertl.hpp>
+#include <boost/spirit/include/lex_lexertl_position_token.hpp>
+#include <boost/spirit/include/phoenix_object.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+#include <boost/spirit/include/phoenix_container.hpp>
+#include <boost/spirit/include/qi_numeric.hpp>
+
+namespace spirit = boost::spirit;
+namespace lex = boost::spirit::lex;
+namespace phoenix = boost::phoenix;
+namespace mpl = boost::mpl;
+
+///////////////////////////////////////////////////////////////////////////////
+enum tokenids
+{
+    ID_INT = 1000,
+    ID_DOUBLE
+};
+
+template <typename Lexer>
+struct token_definitions : lex::lexer<Lexer>
+{
+    token_definitions()
+    {
+        this->self.add_pattern("HEXDIGIT", "[0-9a-fA-F]");
+        this->self.add_pattern("OCTALDIGIT", "[0-7]");
+        this->self.add_pattern("DIGIT", "[0-9]");
+
+        this->self.add_pattern("OPTSIGN", "[-+]?");
+        this->self.add_pattern("EXPSTART", "[eE][-+]");
+        this->self.add_pattern("EXPONENT", "[eE]{OPTSIGN}{DIGIT}+");
+
+        // define tokens and associate them with the lexer
+        int_ = "{OPTSIGN}[1-9]{DIGIT}*";
+        int_.id(ID_INT);
+
+        double_ = "{OPTSIGN}({DIGIT}*\\.{DIGIT}+|{DIGIT}+\\.){EXPONENT}?|{DIGIT}+{EXPONENT}";
+        double_.id(ID_DOUBLE);
+
+        whitespace = "[ \t\n]+";
+
+        this->self = 
+                double_ 
+            |   int_ 
+            |   whitespace[ lex::_pass = lex::pass_flags::pass_ignore ]
+            ;
+    }
+
+    lex::token_def<double> int_;
+    lex::token_def<double> double_;
+    lex::token_def<lex::omit> whitespace;
+};
+
+template <typename Lexer>
+struct token_definitions_with_state : lex::lexer<Lexer>
+{
+    token_definitions_with_state()
+    {
+        this->self.add_pattern("HEXDIGIT", "[0-9a-fA-F]");
+        this->self.add_pattern("OCTALDIGIT", "[0-7]");
+        this->self.add_pattern("DIGIT", "[0-9]");
+
+        this->self.add_pattern("OPTSIGN", "[-+]?");
+        this->self.add_pattern("EXPSTART", "[eE][-+]");
+        this->self.add_pattern("EXPONENT", "[eE]{OPTSIGN}{DIGIT}+");
+
+        this->self.add_state();
+        this->self.add_state("INT");
+        this->self.add_state("DOUBLE");
+
+        // define tokens and associate them with the lexer
+        int_ = "{OPTSIGN}[1-9]{DIGIT}*";
+        int_.id(ID_INT);
+
+        double_ = "{OPTSIGN}({DIGIT}*\\.{DIGIT}+|{DIGIT}+\\.){EXPONENT}?|{DIGIT}+{EXPONENT}";
+        double_.id(ID_DOUBLE);
+
+        whitespace = "[ \t\n]+";
+
+        this->self("*") = 
+                double_ [ lex::_state = "DOUBLE"] 
+            |   int_ [ lex::_state = "INT" ]
+            |   whitespace[ lex::_pass = lex::pass_flags::pass_ignore ]
+            ;
+    }
+
+    lex::token_def<double> int_;
+    lex::token_def<double> double_;
+    lex::token_def<lex::omit> whitespace;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+template <typename Token>
+inline bool 
+test_token_ids(int const* ids, std::vector<Token> const& tokens)
+{
+    BOOST_FOREACH(Token const& t, tokens)
+    {
+        if (*ids == -1)
+            return false;           // reached end of expected data
+
+        if (t.id() != static_cast<std::size_t>(*ids))        // token id must match
+            return false;
+
+        ++ids;
+    }
+
+    return (*ids == -1) ? true : false;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+template <typename Token>
+inline bool 
+test_token_states(std::size_t const* states, std::vector<Token> const& tokens)
+{
+    BOOST_FOREACH(Token const& t, tokens)
+    {
+        if (*states == std::size_t(-1))
+            return false;           // reached end of expected data
+
+        if (t.state() != *states)            // token state must match
+            return false;
+
+        ++states;
+    }
+
+    return (*states == std::size_t(-1)) ? true : false;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+struct position_type
+{
+    std::size_t begin, end;
+};
+
+template <typename Iterator, typename Token>
+inline bool 
+test_token_positions(Iterator begin, position_type const* positions, 
+    std::vector<Token> const& tokens)
+{
+    BOOST_FOREACH(Token const& t, tokens)
+    {
+        if (positions->begin == std::size_t(-1) && 
+            positions->end == std::size_t(-1))
+        {
+            return false;           // reached end of expected data
+        }
+
+        boost::iterator_range<Iterator> matched = t.matched();
+        std::size_t start = std::distance(begin, matched.begin());
+        std::size_t end = std::distance(begin, matched.end());
+
+        // position must match
+        if (start != positions->begin || end != positions->end)
+            return false;
+
+        ++positions;
+    }
+
+    return (positions->begin == std::size_t(-1) && 
+            positions->end == std::size_t(-1)) ? true : false;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+template <typename Token>
+inline bool 
+test_token_values(double const* values, std::vector<Token> const& tokens)
+{
+    BOOST_FOREACH(Token const& t, tokens)
+    {
+        if (*values == 0.0)
+            return false;               // reached end of expected data
+
+        double val;
+        spirit::traits::assign_to(t, val);
+        if (val != *values)             // token value must match
+            return false;
+
+        ++values;
+    }
+
+    return (*values == 0.0) ? true : false;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+int main()
+{
+    typedef std::string::iterator base_iterator_type;
+    std::string input("  1 1.2 -2   3 2.3e6 -3.4");
+    int ids[] = { ID_INT, ID_DOUBLE, ID_INT, ID_INT, ID_DOUBLE, ID_DOUBLE, -1 };
+    std::size_t states[] = { 0, 1, 2, 1, 1, 2, std::size_t(-1) };
+    position_type positions[] = 
+    {
+        { 2, 3 }, { 4, 7 }, { 8, 10 }, { 13, 14 }, { 15, 20 }, { 21, 25 }, 
+        { std::size_t(-1), std::size_t(-1) }
+    };
+    double values[] = { 1.0, 1.2, -2.0, 3.0, 2.3e6, -3.4, 0.0 };
+
+    // token type: token id, iterator_pair as token value, no state
+    {
+        typedef lex::lexertl::token<
+            base_iterator_type, mpl::vector<double>, mpl::false_> token_type;
+        typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+        token_definitions<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        BOOST_TEST(test_token_ids(ids, tokens));
+        BOOST_TEST(test_token_values(values, tokens));
+    }
+
+    {
+        typedef lex::lexertl::position_token<
+            base_iterator_type, mpl::vector<double>, mpl::false_> token_type;
+        typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+        token_definitions<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        BOOST_TEST(test_token_ids(ids, tokens));
+        BOOST_TEST(test_token_positions(input.begin(), positions, tokens));
+        BOOST_TEST(test_token_values(values, tokens));
+    }
+
+    // token type: holds token id, state, iterator_pair as token value
+    {
+        typedef lex::lexertl::token<
+            base_iterator_type, mpl::vector<double>, mpl::true_> token_type;
+        typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+        token_definitions_with_state<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        BOOST_TEST(test_token_ids(ids, tokens));
+        BOOST_TEST(test_token_states(states, tokens));
+        BOOST_TEST(test_token_values(values, tokens));
+    }
+
+    {
+        typedef lex::lexertl::position_token<
+            base_iterator_type, mpl::vector<double>, mpl::true_> token_type;
+        typedef lex::lexertl::actor_lexer<token_type> lexer_type;
+
+        token_definitions_with_state<lexer_type> lexer;
+        std::vector<token_type> tokens;
+        base_iterator_type first = input.begin();
+
+        using phoenix::arg_names::_1;
+        BOOST_TEST(lex::tokenize(first, input.end(), lexer
+          , phoenix::push_back(phoenix::ref(tokens), _1)));
+
+        BOOST_TEST(test_token_ids(ids, tokens));
+        BOOST_TEST(test_token_states(states, tokens));
+        BOOST_TEST(test_token_positions(input.begin(), positions, tokens));
+        BOOST_TEST(test_token_values(values, tokens));
+    }
+
+    return boost::report_errors();
+}