7 files changed, 297 insertions, 0 deletions
diff --git a/src/boost/libs/tokenizer/test/Jamfile.v2 b/src/boost/libs/tokenizer/test/Jamfile.v2
new file mode 100644
index 00000000..58bc06fb
--- /dev/null
+++ b/src/boost/libs/tokenizer/test/Jamfile.v2
@@ -0,0 +1,15 @@
+#~ Copyright Rene Rivera 2008
+#~ Distributed under the Boost Software License, Version 1.0.
+#~ (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+
+import testing ;
+
+test-suite tokenizer
+:   [ run examples.cpp
+        /boost/test//boost_test_exec_monitor/<link>static ]
+    [ run simple_example_1.cpp ]
+    [ run simple_example_2.cpp ]
+    [ run simple_example_3.cpp ]
+    [ run simple_example_4.cpp ]
+    [ run simple_example_5.cpp ]
+    ;
diff --git a/src/boost/libs/tokenizer/test/examples.cpp b/src/boost/libs/tokenizer/test/examples.cpp
new file mode 100644
index 00000000..1e0b69b8
--- /dev/null
+++ b/src/boost/libs/tokenizer/test/examples.cpp
@@ -0,0 +1,150 @@
+// Boost tokenizer examples  -------------------------------------------------//
+
+// (c) Copyright John R. Bandela 2001. 
+
+// Distributed under the Boost Software License, Version 1.0. (See
+// accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// See http://www.boost.org for updates, documentation, and revision history.
+
+#include <iostream>
+#include <iterator>
+#include <string>
+#include <algorithm>
+#include <boost/tokenizer.hpp>
+#include <boost/array.hpp>
+
+#include <boost/test/minimal.hpp>
+
+int test_main( int /*argc*/, char* /*argv*/[] )
+{
+  using namespace boost;
+
+  // Use tokenizer
+  {
+    const std::string test_string = ";;Hello|world||-foo--bar;yow;baz|";
+    std::string answer[] = { "Hello", "world",  "foo", "bar", "yow",  "baz" };
+    typedef tokenizer<char_separator<char> > Tok;
+    char_separator<char> sep("-;|");
+    Tok t(test_string, sep);
+    BOOST_REQUIRE(std::equal(t.begin(),t.end(),answer));
+  }
+  {
+    const std::string test_string = ";;Hello|world||-foo--bar;yow;baz|";
+    std::string answer[] = { "", "", "Hello", "|", "world", "|", "", "|", "",
+                                            "foo", "", "bar", "yow", "baz", "|", "" };
+    typedef tokenizer<char_separator<char> > Tok;
+    char_separator<char> sep("-;", "|", boost::keep_empty_tokens);
+    Tok t(test_string, sep);
+    BOOST_REQUIRE(std::equal(t.begin(), t.end(), answer));
+  }
+  {
+    const std::string test_string = "This,,is, a.test..";
+    std::string answer[] = {"This","is","a","test"};
+    typedef tokenizer<> Tok;
+    Tok t(test_string);
+    BOOST_REQUIRE(std::equal(t.begin(),t.end(),answer));
+  }
+
+  {
+    const std::string test_string = "Field 1,\"embedded,comma\",quote \\\", escape \\\\";
+    std::string answer[] = {"Field 1","embedded,comma","quote \""," escape \\"};
+    typedef tokenizer<escaped_list_separator<char> > Tok;
+    Tok t(test_string);
+    BOOST_REQUIRE(std::equal(t.begin(),t.end(),answer));
+  }
+
+  {
+    const std::string test_string = ",1,;2\\\";3\\;,4,5^\\,\'6,7\';";
+    std::string answer[] = {"","1","","2\"","3;","4","5\\","6,7",""};
+    typedef tokenizer<escaped_list_separator<char> > Tok;
+    escaped_list_separator<char> sep("\\^",",;","\"\'");
+    Tok t(test_string,sep);
+    BOOST_REQUIRE(std::equal(t.begin(),t.end(),answer));
+  }
+
+  {
+    const std::string test_string = "12252001";
+    std::string answer[] = {"12","25","2001"};
+    typedef tokenizer<offset_separator > Tok;
+    boost::array<int,3> offsets = {{2,2,4}};
+    offset_separator func(offsets.begin(),offsets.end());
+    Tok t(test_string,func);
+    BOOST_REQUIRE(std::equal(t.begin(),t.end(),answer));
+  }
+
+  // Use token_iterator_generator
+  {
+    const std::string test_string = "This,,is, a.test..";
+    std::string answer[] = {"This","is","a","test"};
+    typedef token_iterator_generator<char_delimiters_separator<char> >::type Iter;
+    Iter begin = make_token_iterator<std::string>(test_string.begin(),
+      test_string.end(),char_delimiters_separator<char>());
+    Iter end;
+    BOOST_REQUIRE(std::equal(begin,end,answer));
+  }
+
+  {
+    const std::string test_string = "Field 1,\"embedded,comma\",quote \\\", escape \\\\";
+    std::string answer[] = {"Field 1","embedded,comma","quote \""," escape \\"};
+    typedef token_iterator_generator<escaped_list_separator<char> >::type Iter;
+    Iter begin = make_token_iterator<std::string>(test_string.begin(),
+      test_string.end(),escaped_list_separator<char>());
+    Iter begin_c(begin);
+    Iter end;
+    BOOST_REQUIRE(std::equal(begin,end,answer));
+
+    while(begin_c != end)
+    {
+       BOOST_REQUIRE(begin_c.at_end() == 0);
+       ++begin_c;
+    }
+    BOOST_REQUIRE(begin_c.at_end());
+  }
+
+  {
+    const std::string test_string = "12252001";
+    std::string answer[] = {"12","25","2001"};
+    typedef token_iterator_generator<offset_separator>::type Iter;
+    boost::array<int,3> offsets = {{2,2,4}};
+    offset_separator func(offsets.begin(),offsets.end());
+    Iter begin = make_token_iterator<std::string>(test_string.begin(),
+      test_string.end(),func);
+    Iter end= make_token_iterator<std::string>(test_string.end(),
+      test_string.end(),func);
+    BOOST_REQUIRE(std::equal(begin,end,answer));
+  }
+
+  // Test copying
+  {
+    const std::string test_string = "abcdef";
+    token_iterator_generator<offset_separator>::type beg, end, other;
+    boost::array<int,3> ar = {{1,2,3}};
+    offset_separator f(ar.begin(),ar.end());
+    beg = make_token_iterator<std::string>(test_string.begin(),test_string.end(),f);
+
+    ++beg;
+    other = beg;
+    ++other;
+
+    BOOST_REQUIRE(*beg=="bc");
+    BOOST_REQUIRE(*other=="def");
+
+    other = make_token_iterator<std::string>(test_string.begin(),
+        test_string.end(),f);
+
+    BOOST_REQUIRE(*other=="a");
+  }
+
+  // Test non-default constructed char_delimiters_separator
+  {
+    const std::string test_string = "how,are you, doing";
+    std::string answer[] = {"how",",","are you",","," doing"};
+    tokenizer<> t(test_string,char_delimiters_separator<char>(true,",",""));
+    BOOST_REQUIRE(std::equal(t.begin(),t.end(),answer));
+  }
+
+  return 0;
+}
+
diff --git a/src/boost/libs/tokenizer/test/simple_example_1.cpp b/src/boost/libs/tokenizer/test/simple_example_1.cpp
new file mode 100644
index 00000000..950f7328
--- /dev/null
+++ b/src/boost/libs/tokenizer/test/simple_example_1.cpp
@@ -0,0 +1,25 @@
+// (c) Copyright John R. Bandela 2001. 
+
+// Distributed under the Boost Software License, Version 1.0. (See
+// accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// See http://www.boost.org/libs/tokenizer for documenation
+
+
+// simple_example_1.cpp
+#include<iostream>
+#include<boost/tokenizer.hpp>
+#include<string>
+
+int main(){
+   using namespace std;
+   using namespace boost;
+   string s = "This is,  a test";
+   tokenizer<> tok(s);
+   for(tokenizer<>::iterator beg=tok.begin(); beg!=tok.end();++beg){
+       cout << *beg << "\n";
+   }
+   return 0;
+}
+
diff --git a/src/boost/libs/tokenizer/test/simple_example_2.cpp b/src/boost/libs/tokenizer/test/simple_example_2.cpp
new file mode 100644
index 00000000..1bac2d9b
--- /dev/null
+++ b/src/boost/libs/tokenizer/test/simple_example_2.cpp
@@ -0,0 +1,24 @@
+// (c) Copyright John R. Bandela 2001. 
+
+// Distributed under the Boost Software License, Version 1.0. (See
+// accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// See http://www.boost.org/libs/tokenizer for documenation
+
+// simple_example_2.cpp
+#include<iostream>
+#include<boost/tokenizer.hpp>
+#include<string>
+
+int main(){
+   using namespace std;
+   using namespace boost;
+   string s = "Field 1,\"putting quotes around fields, allows commas\",Field 3";
+   tokenizer<escaped_list_separator<char> > tok(s);
+   for(tokenizer<escaped_list_separator<char> >::iterator beg=tok.begin(); beg!=tok.end();++beg){
+       cout << *beg << "\n";
+   }
+   return 0;
+}
+
diff --git a/src/boost/libs/tokenizer/test/simple_example_3.cpp b/src/boost/libs/tokenizer/test/simple_example_3.cpp
new file mode 100644
index 00000000..6f47d353
--- /dev/null
+++ b/src/boost/libs/tokenizer/test/simple_example_3.cpp
@@ -0,0 +1,25 @@
+// (c) Copyright John R. Bandela 2001. 
+
+// Distributed under the Boost Software License, Version 1.0. (See
+// accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// See http://www.boost.org/libs/tokenizer for documenation
+
+// simple_example_3.cpp
+#include<iostream>
+#include<boost/tokenizer.hpp>
+#include<string>
+
+int main(){
+   using namespace std;
+   using namespace boost;
+   string s = "12252001";
+   int offsets[] = {2,2,4};
+   offset_separator f(offsets, offsets+3);
+   tokenizer<offset_separator> tok(s,f);
+   for(tokenizer<offset_separator>::iterator beg=tok.begin(); beg!=tok.end();++beg){
+       cout << *beg << "\n";
+   }
+   return 0;
+}
diff --git a/src/boost/libs/tokenizer/test/simple_example_4.cpp b/src/boost/libs/tokenizer/test/simple_example_4.cpp
new file mode 100644
index 00000000..cf7ead39
--- /dev/null
+++ b/src/boost/libs/tokenizer/test/simple_example_4.cpp
@@ -0,0 +1,24 @@
+// (c) Copyright John R. Bandela 2001. 
+
+// Distributed under the Boost Software License, Version 1.0. (See
+// accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// See http://www.boost.org/libs/tokenizer for documenation
+
+// simple_example_4.cpp
+#include<iostream>
+#include<boost/tokenizer.hpp>
+#include<string>
+
+int main(){
+   using namespace std;
+   using namespace boost;
+   string s = "This is,  a test";
+   tokenizer<char_delimiters_separator<char> > tok(s);
+   for(tokenizer<char_delimiters_separator<char> >::iterator beg=tok.begin(); beg!=tok.end();++beg){
+       cout << *beg << "\n";
+   }
+   return 0;
+}
+
diff --git a/src/boost/libs/tokenizer/test/simple_example_5.cpp b/src/boost/libs/tokenizer/test/simple_example_5.cpp
new file mode 100644
index 00000000..8259533c
--- /dev/null
+++ b/src/boost/libs/tokenizer/test/simple_example_5.cpp
@@ -0,0 +1,34 @@
+// (c) Copyright John R. Bandela 2001. 
+
+// Distributed under the Boost Software License, Version 1.0. (See
+// accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+// See http://www.boost.org/libs/tokenizer for documenation
+
+/// simple_example_5.cpp
+#include<iostream>
+#include<boost/token_iterator.hpp>
+#include<string>
+
+#ifdef __BORLANDC__
+// compiler bug fix:
+template class boost::token_iterator_generator<boost::offset_separator>::type;
+#endif
+
+int main(){
+   using namespace std;
+   using namespace boost;
+   string s = "12252001";
+   int offsets[] = {2,2,4};
+   offset_separator f(offsets, offsets+3);
+   typedef token_iterator_generator<offset_separator>::type Iter;
+   Iter beg = make_token_iterator<string>(s.begin(),s.end(),f);
+   Iter end = make_token_iterator<string>(s.end(),s.end(),f); 
+   // The above statement could also have been what is below
+   // Iter end;
+   for(;beg!=end;++beg){
+     cout << *beg << "\n";
+   }
+   return 0;
+}