10 files changed, 520 insertions, 0 deletions
diff --git a/bin/build-extra-dist.sh b/bin/build-extra-dist.sh
new file mode 100755
index 0000000..3c76312
--- /dev/null
+++ b/bin/build-extra-dist.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+#
+# Script to build lists of extra dist files by category.
+# Output of this script should go into Makefile.am at the project root.
+
+function print_list()
+{
+    local _DIR=$1
+    local _NAME=$_DIR"_data"
+
+    echo "$_NAME = \\"
+    local _BASE_CMD="git ls-tree --full-tree --name-only -r HEAD -- $_DIR"
+    $_BASE_CMD | sed -e 's/\#/\\\#/g' | head -n -1 | sed -e 's/^/\t/g' -e 's/$/\ \\/g'
+    $_BASE_CMD | sed -e 's/\#/\\\#/g' | tail -n 1 | sed -e 's/^/\t/g'
+    echo ""
+}
+
+PROGDIR=`dirname $0`
+cd "../$PROGDIR"
+
+print_list doc
+print_list doc_example
+print_list bin
+print_list misc
+print_list slickedit
+print_list test
diff --git a/bin/env-osx.sh b/bin/env-osx.sh
new file mode 100644
index 0000000..a7c2e8e
--- /dev/null
+++ b/bin/env-osx.sh
@@ -0,0 +1,16 @@
+
+# Source this to set the debug environment for OSX.
+
+PROGDIR=`dirname $0`
+ROOTDIR="$PROGDIR/.."
+
+PARSER_DIR="$ROOTDIR/src/parser/.libs"
+LIBORCUS_DIR="$ROOTDIR/src/liborcus/.libs"
+SPREADSHEET_DIR="$ROOTDIR/src/spreadsheet/.libs"
+
+LD_PATH="$PARSER_DIR:$LIBORCUS_DIR:$SPREADSHEET_DIR"
+
+export DYLD_LIBRARY_PATH="$LD_PATH"
+
+
+
diff --git a/bin/gen-cli-docs.py b/bin/gen-cli-docs.py
new file mode 100755
index 0000000..39e8a72
--- /dev/null
+++ b/bin/gen-cli-docs.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+
+import argparse
+import subprocess
+import io
+import os
+import copy
+from pathlib import Path
+
+
+def _print_option(writable, option, description):
+    line = ' '.join(option)
+    print(f"- ``{line}``", file=writable)
+    print(file=writable)
+
+    line_buf = list()
+    description_cleaned = list()
+    for line in description:
+        if not line_buf:
+            line_buf = copy.deepcopy(line)
+            continue
+
+        if not line:
+            # empty line
+            if line_buf:
+                description_cleaned.append(line_buf)
+            description_cleaned.append(line)
+            line_buf = list()
+            continue
+
+        if line[0] == '*':
+            description_cleaned.append(line_buf)
+            line_buf = list()
+
+        line_buf.extend(line)
+
+    if line_buf:
+        description_cleaned.append(line_buf)
+
+    for line in description_cleaned:
+        if line and line[0] == '*':
+            line[0] = '-'
+            line = "  " + ' '.join(line)
+            print(f"  {line}", file=writable)
+            continue
+
+        print("  " + ' '.join(line), file=writable)
+
+    print(file=writable)
+
+
+def _parse_and_print(writable, cmd_name, lines):
+    print(cmd_name, file=writable)
+    print('=' * len(cmd_name), file=writable)
+    print(file=writable)
+
+    # Usage (1st line)
+    line = lines[0]
+    prefix = "Usage: "
+    if not line.startswith(prefix):
+        raise RuntimeError("invalid output")
+
+    cmd = line[len(prefix):]
+    print("Usage", file=writable)
+    print("-----", file=writable)
+    print(file=writable)
+    print(".. code-block::", file=writable)
+    print(file=writable)
+    print(f"   {cmd}", file=writable)
+    print(file=writable)
+
+    # Description (sentence block right below usage)
+    for i, line in enumerate(lines[2:]):
+        if line == "Options:":
+            break
+        print(line, file=writable)
+
+    # Allowed option title
+    lineno = i + 2
+    line = lines[lineno]
+    if not line or line[-1] != ':':
+        print(lines)
+        raise RuntimeError("invalid section title")
+
+    line = line[:-1]
+    print(line, file=writable)
+    print('-' * len(line), file=writable)
+    print(file=writable)
+
+    # Options
+    lineno += 1
+
+    # determine the first indent length from the first line.
+    indent = 0
+    while lines[lineno][indent] == ' ':
+        indent += 1
+
+    # determine the column position of the option description.
+    desc_pos = lines[lineno].find("Print this help.")
+    if desc_pos < 0:
+        raise RuntimeError("failed to parse the --help option line.")
+
+    option_buf = list()
+    desc_buf = list()
+    for line in lines[lineno:]:
+        if not line:
+            continue
+
+        option_s = line[indent:desc_pos]
+        desc_s = line[desc_pos:]
+
+        if option_s and option_s[0] == '-':
+            # start a new option. if the current buffer is not empty, flush it first.
+            if option_buf:
+                _print_option(writable, option_buf, desc_buf)
+
+            option_buf = option_s.split()
+            desc_buf = list()
+
+        desc_buf.append(desc_s.split())
+
+    if option_buf:
+        _print_option(writable, option_buf, desc_buf)
+
+
+def parse(cmd_dir, output_dir, cmd_name):
+    os.makedirs(output_dir, exist_ok=True)
+    cmd_path = cmd_dir / cmd_name
+    s = cmd_name.replace('-', '_')
+    output_path = output_dir / f"{s}.rst"
+    if not cmd_path.is_file():
+        raise RuntimeError(f"command not found: {cmd_path}")
+
+    output = subprocess.run([cmd_path, "-h"], stdout=subprocess.PIPE).stdout
+    output = output.decode("utf-8")
+    lines = output.split('\n')
+
+    with open(output_path, "w") as f:
+        _parse_and_print(f, cmd_name, lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Parse the output from the cli help, and convert it to rst output.")
+    parser.add_argument(
+        "--cmd-dir", "-c", type=Path, required=True,
+        help="path to the directory where the orcus commands are.")
+    parser.add_argument(
+        "--output-dir", "-o", type=Path, required=True,
+        help="path to the output directory.")
+    args = parser.parse_args()
+
+    if not args.cmd_dir.is_dir():
+        raise RuntimeError(f"invalid command directory: {args.cmd_dir}")
+
+    cmds = (
+        "orcus-csv",
+        "orcus-ods",
+        "orcus-xlsx",
+        "orcus-gnumeric",
+        "orcus-json",
+        "orcus-xml",
+        "orcus-xls-xml",
+        "orcus-yaml",
+        "orcus-parquet",
+    )
+    for cmd in cmds:
+        parse(args.cmd_dir, args.output_dir, cmd)
+
+
+if __name__ == "__main__":
+    main()
+
diff --git a/bin/gen-files.py b/bin/gen-files.py
new file mode 100644
index 0000000..5684277
--- /dev/null
+++ b/bin/gen-files.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""Script to generate files from the .in files the same way autoconf does.
+
+It is to be used only when autoconf is not used to do the build.
+"""
+
+import argparse
+import os.path
+import sys
+
+
+class FileGenerator(object):
+
+    @staticmethod
+    def parse_properties(props):
+        d = dict()
+        for kv in props:
+            key, value = kv.split('=')
+            d[key] = value
+        return d
+
+    def __init__(self, props, outfiles):
+        self.__props = FileGenerator.parse_properties(props)
+        self.__outfiles = outfiles
+
+    def run(self):
+        for outfile in self.__outfiles:
+            infile = "{}.in".format(outfile)
+            print(infile)
+            print(outfile)
+            with open(infile, 'r') as f:
+                content = f.read()
+                self.__write_file(outfile, content)
+
+    def __write_file(self, outfile, src_content):
+        with open(outfile, 'w') as f:
+            keyword_pos = None
+            content_pos = 0
+            for i, c in enumerate(src_content):
+                if c != '@':
+                    continue
+
+                if keyword_pos is None:
+                    # new keyword detected.
+                    keyword_pos = i
+                    # flush content to the destination file.
+                    f.write(src_content[content_pos:i])
+                else:
+                    # keyword span has just ended.
+                    key = src_content[keyword_pos+1:i]
+                    keyword_pos = None
+                    # perform keyword substitution.
+                    if key not in self.__props:
+                        raise RuntimeError("value for {} not defined!".format(key))
+                    f.write(self.__props[key])
+                    content_pos = i + 1
+
+            if keyword_pos is not None:
+                raise RuntimeError("malformed template file!")
+
+            # flush the rest of the content to the destination file.
+            f.write(src_content[content_pos:])
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate files from the .in files.")
+    parser.add_argument(
+        "--properties", type=str, nargs="+",
+        required=True,
+        help="Comma-separated strings representing multiple key-value pairs.")
+    parser.add_argument(
+        "--files", type=str, nargs="+",
+        help="""Source template files from which to generate final files. Each
+        file is expected to end with '.in' but you should not include the '.in'
+        suffix in the argument to this script.""")
+
+    args = parser.parse_args()
+    generator = FileGenerator(args.properties, args.files)
+    generator.run()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/bin/gen-gfm-release-table.sh b/bin/gen-gfm-release-table.sh
new file mode 100755
index 0000000..306c9a9
--- /dev/null
+++ b/bin/gen-gfm-release-table.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+PROJ_PREFIX=orcus
+PKG_PREFIX=lib$PROJ_PREFIX
+
+# Pick up the version number string from configure.ac.
+VER_MAJOR=$(cat ./configure.ac | grep -E $PROJ_PREFIX"_major_version.*[0-9]" | sed -e "s/.*\[\([0-9][0-9]*\).*/\1/g")
+VER_MINOR=$(cat ./configure.ac | grep -E $PROJ_PREFIX"_minor_version.*[0-9]" | sed -e "s/.*\[\([0-9][0-9]*\).*/\1/g")
+VER_MICRO=$(cat ./configure.ac | grep -E $PROJ_PREFIX"_micro_version.*[0-9]" | sed -e "s/.*\[\([0-9][0-9]*\).*/\1/g")
+VER="$VER_MAJOR.$VER_MINOR.$VER_MICRO"
+
+PKGS=$(ls $PKG_PREFIX-$VER.tar.*)
+
+echo "## Release Notes"
+echo ""
+echo "* add item"
+echo ""
+echo "## Source packages for distribution"
+echo ""
+
+echo "| URL | sha256sum | size |"
+echo "|-----|-----------|------|"
+
+for _PKG in $PKGS; do
+    _URL="[$_PKG](https://kohei.us/files/$PROJ_PREFIX/src/$_PKG)"
+    _HASH=$(sha256sum $_PKG | sed -e "s/^\(.*\)$PKG_PREFIX.*/\1/g" | tr -d "[:space:]")
+    _SIZE=$(stat -c "%s" $_PKG)
+    echo "| $_URL | $_HASH | $_SIZE |"
+done
+
diff --git a/bin/gen-test-json-structure-outputs.sh b/bin/gen-test-json-structure-outputs.sh
new file mode 100755
index 0000000..afe0181
--- /dev/null
+++ b/bin/gen-test-json-structure-outputs.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+PROGDIR=`dirname $0`
+TESTDIR="$PROGDIR/../test/json-structure"
+ORCUS_JSON_EXEC="$PROGDIR/../src/orcus-json"
+
+for _DIR in $(ls $TESTDIR); do
+    echo "processing $_DIR..."
+    _DIR="$TESTDIR/$_DIR"
+    _INPUT="$_DIR/input.json"
+    _OUTPUT="$_DIR/check.txt"
+    $ORCUS_JSON_EXEC --mode structure -o "$_OUTPUT" "$_INPUT"
+done
+
diff --git a/bin/gen-test-xml-outputs.sh b/bin/gen-test-xml-outputs.sh
new file mode 100755
index 0000000..137b20c
--- /dev/null
+++ b/bin/gen-test-xml-outputs.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+PROGDIR=`dirname $0`
+TESTDIR="$PROGDIR/../test/xml"
+ORCUS_XML_EXEC="$PROGDIR/../src/orcus-xml"
+
+for _DIR in $(ls $TESTDIR); do
+    _DIR="$TESTDIR/$_DIR"
+    _INPUT="$_DIR/input.xml"
+    _OUTPUT="$_DIR/check.txt"
+
+    if [ ! -f "$_INPUT" ]; then
+        continue
+    fi
+
+    echo "processing $_DIR..."
+    $ORCUS_XML_EXEC --mode dump -o "$_OUTPUT" "$_INPUT"
+done
+
diff --git a/bin/run-bulk-tests.py b/bin/run-bulk-tests.py
new file mode 100755
index 0000000..a2dd6b1
--- /dev/null
+++ b/bin/run-bulk-tests.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+########################################################################
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+########################################################################
+
+import argparse
+import io
+import os
+import subprocess
+from pathlib import Path
+
+
+def main():
+    desc = """Tool to bulk-test a number of files and collect all their results
+    into a single output directory."""
+
+    parser = argparse.ArgumentParser(description=desc)
+    parser.add_argument("--args", type=str, required=True)
+    parser.add_argument("--ext", "-e", type=str, required=True, help="File extension to use.")
+    parser.add_argument("--outdir", "-o", type=Path, required=True, help="Output directory to put the run results in.")
+    parser.add_argument("dirs", nargs='*', type=Path)
+    args = parser.parse_args()
+
+    args.outdir.joinpath("good").mkdir(parents=True, exist_ok=True)
+    args.outdir.joinpath("bad").mkdir(parents=True, exist_ok=True)
+
+    ext = args.ext
+    if ext and ext[0] != '.':
+        ext = f".{ext}"
+
+    count = 0
+    for parent_dir in args.dirs:
+        for rootdir, _, files in os.walk(parent_dir):
+            for f in files:
+                filepath = Path(rootdir) / f
+                if filepath.suffix != ext:
+                    continue
+
+                buf = io.StringIO()
+                cmd = args.args.split()
+                cmd.append(str(filepath))
+                res = subprocess.run(cmd, capture_output=True)
+                outpath = args.outdir
+                outpath /= "good" if res.returncode == 0 else "bad"
+                outpath /= f"{count:04d}.txt"
+                print("-- command --", file=buf)
+                print(" ".join(cmd), file=buf)
+                print("-- return-code --", file=buf)
+                print(res.returncode, file=buf)
+                print("-- stdout --", file=buf)
+                print(res.stdout.decode("utf-8"), file=buf)
+                print("-- stderr --", file=buf)
+                print(res.stderr.decode("utf-8"), file=buf)
+                outpath.write_text(buf.getvalue())
+                count += 1
+
+
+if __name__ == "__main__":
+    main()
diff --git a/bin/run-python.sh b/bin/run-python.sh
new file mode 100755
index 0000000..f3d314a
--- /dev/null
+++ b/bin/run-python.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+PYTHON=$(which python3)
+PROGDIR=$(dirname $0)
+_PYTHONPATH="$PROGDIR/../src/python/.libs:$PROGDIR/../src/python"
+
+export PYTHONPATH=$_PYTHONPATH:$PYTHONPATH
+export LD_LIBRARY_PATH="$PROGDIR/../src/liborcus/.libs:$PROGDIR/../src/parser/.libs"
+export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH
+
+if [ "$1" == "" ]; then
+    # No input file. Just invoke the interpreter.
+    $PYTHON
+    exit 0
+fi
+
+RUNMODE=
+
+if [ "$1" == "gdb" ]; then
+    RUNMODE=gdb
+    shift
+elif [ "$1" == "valgrind" ]; then
+    RUNMODE=valgrind
+    shift
+fi
+
+if [ ! -e "$1" ]; then
+    echo "file '$1' does not exist"
+    exit 1
+fi
+
+EXEC="$1"
+shift
+
+case $RUNMODE in
+    gdb)
+    gdb --args $PYTHON "$PWD/$EXEC" "$@"
+        ;;
+    valgrind)
+    valgrind --tool=memcheck --leak-check=full --show-leak-kinds=all --track-origins=yes $PYTHON "$PWD/$EXEC" "$@"
+        ;;
+    *)
+        exec "$PWD/$EXEC" "$@"
+        ;;
+esac
+
+
diff --git a/bin/unpack-zipped-xml.sh b/bin/unpack-zipped-xml.sh
new file mode 100755
index 0000000..ac22fe9
--- /dev/null
+++ b/bin/unpack-zipped-xml.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+# global constants
+outdir=xml
+
+function abort()
+{
+    (>&2 echo "$1")  # output to stderr
+    exit 1
+}
+
+filepath="$1"
+
+if [ -z "$filepath" ]; then
+    abort "file path is not given."
+fi
+
+shift
+
+# convert the file path to absolute path.
+filepath=`realpath "$filepath"`
+
+# remove existing output directory if one exists.
+if [ -d $outdir ]; then
+    rm -rf $outdir || abort "failed to remove the existing output directory '$outdir'."
+fi
+
+mkdir $outdir || abort "failed to create an output directory '$outdir'."
+
+# unzip all inside the output directory.
+cd $outdir
+unzip "$filepath" > /dev/null || abort "failed to unzip $filepath."
+
+# temporarily replace bash's internal field separators to handle file names with spaces.
+_IFS="$IFS"
+IFS=$'\n'
+
+for _file in $(find . -type f); do
+    _mimetype=`file --mime-type --brief "$_file"` || abort "failed to determine the mime type of $_file."
+    if [ $_mimetype = "application/xml" ] || [ $_mimetype = "text/xml" ]; then
+        # beautify the XML file content.
+        _temp=$(mktemp) || abort "failed to create a temporary file."
+        xmllint --format $_file > $_temp || abort "failed to run xmllint on $_file."
+        mv $_temp $_file || abort "failed to update $_file."
+    fi
+done
+
+# restore the original separators.
+IFS="$_IFS"