diff options
Diffstat (limited to 'storage/mroonga/tools')
| -rw-r--r-- | storage/mroonga/tools/Makefile.am | 6 | ||||
| -rwxr-xr-x | storage/mroonga/tools/prepare-sphinx-html.rb | 175 | ||||
| -rwxr-xr-x | storage/mroonga/tools/travis/before_script.sh | 95 | ||||
| -rwxr-xr-x | storage/mroonga/tools/travis/install.sh | 157 | ||||
| -rwxr-xr-x | storage/mroonga/tools/travis/script.sh | 136 | ||||
| -rwxr-xr-x | storage/mroonga/tools/upload-to-github.rb | 42 |
6 files changed, 611 insertions, 0 deletions
diff --git a/storage/mroonga/tools/Makefile.am b/storage/mroonga/tools/Makefile.am new file mode 100644 index 00000000..b65b9300 --- /dev/null +++ b/storage/mroonga/tools/Makefile.am @@ -0,0 +1,6 @@ +noinstall_ruby_scripts = \ + prepare-sphinx-html.rb \ + upload-to-github.rb + +EXTRA_DIST = \ + $(noinstall_ruby_scripts) diff --git a/storage/mroonga/tools/prepare-sphinx-html.rb b/storage/mroonga/tools/prepare-sphinx-html.rb new file mode 100755 index 00000000..71e12f0e --- /dev/null +++ b/storage/mroonga/tools/prepare-sphinx-html.rb @@ -0,0 +1,175 @@ +#!/usr/bin/env ruby +# -*- coding: utf-8 -*- + +if ARGV.size != 2 + puts "Usage: #{$0} SOURCE_DIR DEST_DIR" + exit(false) +end + +require 'pathname' +require "fileutils" + +def fix_link(text, extension, language) + send("fix_#{extension}_link", text, language) +end + +def fix_link_path(text) + text.gsub(/\b_(sources|static|images)\b/, '\1') +end + +def fix_language_link(url, language) + url.gsub(/\A((?:\.\.\/){2,})([a-z]{2})\/html\//) do + relative_base_path = $1 + link_language = $2 + close_quote = $3 + if language == "en" + relative_base_path = relative_base_path.gsub(/\A\.\.\//, '') + end + if link_language != "en" + relative_base_path += "#{link_language}/" + end + "#{relative_base_path}docs/" + end +end + +def fix_html_link(html, language) + html = html.gsub(/(href|src)="(.+?)"/) do + attribute = $1 + link = $2 + link = fix_link_path(link) + link = fix_language_link(link, language) + "#{attribute}=\"#{link}\"" + end + html.gsub(/(id="top-link" href=)"(.+?)"/) do + prefix = $1 + top_path = $2.gsub(/\/index\.html\z/, '/') + top_path = "./" if ["index.html", "#"].include?(top_path) + "#{prefix}\"#{top_path}../\"" + end +end + +def fix_js_link(js, language) + fix_link_path(js) +end + +LANGUAGE_TO_LOCALE = { + "ja" => "ja_JP", + "en" => "en_US", +} + +def insert_facebook_html_header(html) + html.gsub(/<\/head>/) do + <<-HTML + <meta property="fb:page_id" content="238184682903165" /><!-- mroonga --> + <meta property="fb:admins" content="664204556" /><!-- kouhei.sutou --> + <meta property="og:type" content="product" /> + <meta property="og:image" content="http://mroonga.org/images/logos/mroonga-icon-full-size.png" /> + <meta property="og:site_name" content="mroonga" /> + + <link rel="stylesheet" href="/css/sphinx.css" type="text/css" /> + </head> + HTML + end +end + +def insert_facebook_html_fb_root(html) + html.gsub(/<body>/) do + <<-HTML + <body> + <div id="fb-root"></div> + HTML + end +end + +def insert_facebook_html_buttons(html) + html.gsub(/(<div class="other-language-links">)/) do + <<-HTML + <div class="facebook-buttons"> + <fb:like href="http://www.facebook.com/pages/mroonga/238184682903165" + layout="standard" + width="290"></fb:like> + </div> + #{$1} + HTML + end +end + +def insert_facebook_html_footer(html, language) + locale = LANGUAGE_TO_LOCALE[language] + raise "unknown locale for language #{language.inspect}" if locale.nil? + html.gsub(/<\/body>/) do + <<-HTML + <script src="http://connect.facebook.net/#{locale}/all.js"></script> + + <script> + FB.init({ + appId : null, + status : true, // check login status + cookie : true, // enable cookies to allow the server to access the session + xfbml : true // parse XFBML + }); + </script> + </body> + HTML + end +end + +def insert_facebook_html(html, language) + html = insert_facebook_html_header(html) + html = insert_facebook_html_fb_root(html) + html = insert_facebook_html_buttons(html) + html = insert_facebook_html_footer(html, language) + html +end + +source_dir, dest_dir = ARGV + +source_dir = Pathname.new(source_dir) +dest_dir = Pathname.new(dest_dir) + +language_dirs = [] +source_dir.each_entry do |top_level_path| + language_dirs << top_level_path if /\A[a-z]{2}\z/ =~ top_level_path.to_s +end + +language_dirs.each do |language_dir| + language = language_dir.to_s + language_source_dir = source_dir + language_dir + "html" + language_dest_dir = dest_dir + language_dir + language_source_dir.find do |source_path| + relative_path = source_path.relative_path_from(language_source_dir) + dest_path = language_dest_dir + relative_path + if source_path.directory? + dest_path.mkpath + else + case source_path.extname + when ".html", ".js" + content = source_path.read + extension = source_path.extname.gsub(/\A\./, '') + content = fix_link(content, extension, language) + if extension == "html" + content = insert_facebook_html(content, language) + end + dest_path.open("wb") do |dest| + dest.print(content.strip) + end + FileUtils.touch(dest_path, :mtime => source_path.mtime) + else + case source_path.basename.to_s + when ".buildinfo" + # ignore + else + FileUtils.cp(source_path, dest_path, :preserve => true) + end + end + end + end +end + +dest_dir.find do |dest_path| + if dest_path.directory? and /\A_/ =~ dest_path.basename.to_s + normalized_dest_path = dest_path + ".." + normalized_dest_path += dest_path.basename.to_s.gsub(/\A_/, '') + FileUtils.mv(dest_path, normalized_dest_path) + end +end diff --git a/storage/mroonga/tools/travis/before_script.sh b/storage/mroonga/tools/travis/before_script.sh new file mode 100755 index 00000000..a6491d0c --- /dev/null +++ b/storage/mroonga/tools/travis/before_script.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# +# Copyright(C) 2012-2016 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +# set -x +set -e + +if [ "${MROONGA_BUNDLED}" = "yes" ]; then + cmake_args=(-DCMAKE_BUILD_TYPE=Debug -DWITH_UNIT_TESTS=FALSE) + cmake_args=("${cmake_args[@]}" -DWITH_EMBEDDED_SERVER=TRUE) + cmake_args=("${cmake_args[@]}" -DWITHOUT_ARCHIVE=TRUE) + cmake_args=("${cmake_args[@]}" -DWITHOUT_BLACKHOLE=TRUE) + cmake_args=("${cmake_args[@]}" -DWITHOUT_CONNECT=TRUE) + cmake_args=("${cmake_args[@]}" -DWITHOUT_CSV=TRUE) + cmake_args=("${cmake_args[@]}" -DWITHOUT_EXAMPLE=TRUE) + cmake_args=("${cmake_args[@]}" -DWITHOUT_FEDERATED=TRUE) + cmake_args=("${cmake_args[@]}" -DWITHOUT_FEDERATEDX=TRUE) + cmake_args=("${cmake_args[@]}" -DWITHOUT_HEAP=TRUE) + cmake_args=("${cmake_args[@]}" -DWITHOUT_MYISAMMRG=TRUE) + cmake_args=("${cmake_args[@]}" -DWITHOUT_OQGRAPH=TRUE) + cmake_args=("${cmake_args[@]}" -DWITHOUT_SEQUENCE=TRUE) + cmake_args=("${cmake_args[@]}" -DWITHOUT_SPHINX=TRUE) + cmake_args=("${cmake_args[@]}" -DWITHOUT_SPIDER=TRUE) + cmake_args=("${cmake_args[@]}" -DWITHOUT_TEST_SQL_DISCOVERY=TRUE) + if [ "${MROONGA_TEST_EMBEDDED}" = "yes" ]; then + cmake_args=("${cmake_args[@]}" -DWITH_EMBEDDED_SERVER=TRUE) + cmake_args=("${cmake_args[@]}" -DMRN_BUILD_FOR_EMBEDDED_SERVER=TRUE) + fi + cmake . "${cmake_args[@]}" +else + ./autogen.sh + + if [ -d /opt/mysql/ ]; then + PATH=$(echo /opt/mysql/server-*/bin/):$PATH + fi + configure_args=("--with-mysql-source=$PWD/vendor/mysql") + case "${MYSQL_VERSION}" in + mysql-5.6) + configure_args=("${configure_args[@]}" --enable-fast-mutexes) + ;; + mysql-5.7) + boost_archive=boost_1_59_0.tar.gz + curl -L -O http://downloads.sourceforge.net/project/boost/boost/1.59.0/${boost_archive} + sudo mkdir -p /usr/global/share + sudo mv ${boost_archive} /usr/global/share/ + (cd vendor/mysql && sudo debian/rules override_dh_auto_configure) + ;; + mariadb-5.5) + (cd vendor/mysql && sudo debian/rules configure) + configure_args=("${configure_args[@]}" + "--with-mysql-build=$PWD/vendor/mysql/builddir") + ;; + percona-server-5.6) + (cd vendor/mysql && \ + sudo debian/rules configure SKIP_DEBUG_BINARY=yes && \ + cd builddir/libservices && \ + sudo make > /dev/null && \ + cd ../extra && \ + sudo make > /dev/null) + configure_args=("${configure_args[@]}" + "--enable-fast-mutexes" + "--with-mysql-build=$PWD/vendor/mysql/builddir" + "--with-mysql-config=$PWD/vendor/mysql/builddir/scripts/mysql_config") + ;; + percona-server-5.7) + (cd vendor/mysql && \ + sudo debian/rules override_dh_auto_configure SKIP_DEBUG_BINARY=yes && \ + cd builddir/libservices && \ + sudo make > /dev/null && \ + cd ../extra && \ + sudo make > /dev/null) + configure_args=("${configure_args[@]}" + "--with-mysql-build=$PWD/vendor/mysql/builddir" + "--with-mysql-config=$PWD/vendor/mysql/builddir/scripts/mysql_config") + ;; + *) + : + ;; + esac + ./configure "${configure_args[@]}" +fi diff --git a/storage/mroonga/tools/travis/install.sh b/storage/mroonga/tools/travis/install.sh new file mode 100755 index 00000000..2c8cdbb8 --- /dev/null +++ b/storage/mroonga/tools/travis/install.sh @@ -0,0 +1,157 @@ +#!/bin/sh +# +# Copyright(C) 2012-2017 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +set -x +set -e + +# export GROONGA_MASTER=yes +# export GROONGA_NORMALIZER_MYSQL_MASTER=yes + +#mariadb_download_base=http://mirror.jmu.edu/pub/mariadb +mariadb_download_base=http://ftp.osuosl.org/pub/mariadb + +version=$(echo "$MYSQL_VERSION" | sed -r -e 's/^(mysql|mariadb|percona-server)-//') +series=$(echo "$version" | sed -r -e 's/^([0-9]+\.[0-9]+).*$/\1/g') + +setup_mariadb_apt() +{ + distribution=$(lsb_release --short --id | tr 'A-Z' 'a-z') + code_name=$(lsb_release --short --codename) + component=main + apt_url_base="${mariadb_download_base}/repo/${series}" + cat <<EOF | sudo tee /etc/apt/sources.list.d/mariadb.list +deb ${apt_url_base}/${distribution}/ ${code_name} ${component} +deb-src ${apt_url_base}/${distribution}/ ${code_name} ${component} +EOF + sudo apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 0xcbcb082a1bb943db + sudo apt-get -qq update +} + +setup_percona_apt() +{ + code_name=$(lsb_release --short --codename) + release_deb_version=0.1-4 + release_deb=percona-release_${release_deb_version}.${code_name}_all.deb + wget http://www.percona.com/downloads/percona-release/ubuntu/${release_deb_version}/${release_deb} + sudo dpkg -i ${release_deb} + sudo apt-get -qq update +} + +if [ "${MROONGA_BUNDLED}" = "yes" ]; then + mkdir -p .mroonga + mv * .mroonga/ + mv .mroonga/tools ./ + setup_mariadb_apt + sudo apt-get -qq -y build-dep mariadb-server + # Support MariaDB for now. + download_base=${mariadb_download_base}/${MYSQL_VERSION} + tar_gz=${MYSQL_VERSION}.tar.gz + curl -O ${download_base}/source/${tar_gz} + tar xzf $tar_gz + mv ${MYSQL_VERSION}/* ./ + rm -rf storage/mroonga + mv .mroonga storage/mroonga + rm -rf ${MYSQL_VERSION} + git clone --recursive --depth 1 \ + https://github.com/groonga/groonga.git \ + storage/mroonga/vendor/groonga + git clone --recursive --depth 1 \ + https://github.com/groonga/groonga-normalizer-mysql.git \ + storage/mroonga/vendor/groonga/vendor/plugins/groonga-normalizer-mysql +else + curl --silent --location \ + https://raw.githubusercontent.com/groonga/groonga/master/data/travis/setup.sh | sh + curl --silent --location \ + https://raw.githubusercontent.com/groonga/groonga-normalizer-mysql/master/data/travis/setup.sh | sh + # curl --silent --location \ + # https://raw.githubusercontent.com/clear-code/cutter/master/data/travis/setup.sh | sh + + if [ ! -f /usr/lib/groonga/plugins/tokenizers/mecab.so ]; then + sudo apt-get -qq -y install groonga-tokenizer-mecab + fi + + mkdir -p vendor + cd vendor + + case "$MYSQL_VERSION" in + mysql-*) + sudo apt-get -qq update + sudo apt-get -qq -y build-dep mysql-server + if [ "$version" = "system" ]; then + sudo apt-get -y remove --purge \ + mysql-server-5.6 \ + mysql-server-core-5.6 \ + mysql-client-5.6 \ + mysql-client-core-5.6 + sudo rm -rf /var/lib/mysql + sudo apt-get -y install \ + mysql-server \ + mysql-client \ + mysql-testsuite \ + libmysqld-dev + apt-get source mysql-server + ln -s $(find . -maxdepth 1 -type d | sort | tail -1) mysql + else + repository_deb=mysql-apt-config_0.8.3-1_all.deb + curl -O http://repo.mysql.com/${repository_deb} + sudo env MYSQL_SERVER_VERSION=mysql-${series} \ + dpkg -i ${repository_deb} + sudo apt-get -qq update + sudo apt-get -qq -y remove --purge mysql-common + sudo apt-get -qq -y build-dep mysql-server + sudo apt-get -qq -y install \ + mysql-server \ + libmysqlclient-dev \ + libmysqld-dev \ + mysql-testsuite + apt-get -qq source mysql-server + ln -s $(find . -maxdepth 1 -type d | sort | tail -1) mysql + fi + ;; + mariadb-*) + sudo apt-get -y remove --purge \ + mysql-server-5.6 \ + mysql-server-core-5.6 \ + mysql-client-5.6 \ + mysql-client-core-5.6 \ + mysql-common + sudo rm -rf /var/lib/mysql + setup_mariadb_apt + sudo apt-get -qq -y build-dep mariadb-server + sudo apt-get -y install \ + mariadb-server \ + mariadb-client \ + mariadb-test \ + libmariadbclient-dev + apt-get source mariadb-server + ln -s $(find . -maxdepth 1 -type d | sort | tail -1) mysql + ;; + percona-server-*) + setup_percona_apt + sudo apt-get -qq -y build-dep percona-server-server-${series} + sudo apt-get -qq -y install \ + percona-server-server-${series} \ + percona-server-client-${series} \ + percona-server-test-${series} + apt-get -qq source percona-server-server-${series} + ln -s $(find . -maxdepth 1 -type d | sort | tail -1) mysql + ;; + esac + + cd .. +fi diff --git a/storage/mroonga/tools/travis/script.sh b/storage/mroonga/tools/travis/script.sh new file mode 100755 index 00000000..1c0121e0 --- /dev/null +++ b/storage/mroonga/tools/travis/script.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# +# Copyright(C) 2012-2016 Kouhei Sutou <kou@clear-code.com> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +# set -x +set -e + +base_dir="$(cd $(dirname $0); pwd)" +top_dir="${base_dir}/../.." + +bundled_mroonga_dir="${top_dir}/storage/mroonga" +if [ -f "${bundled_mroonga_dir}/config.sh" ]; then + mroonga_dir="${bundled_mroonga_dir}" + . "${bundled_mroonga_dir}/config.sh" +else + mroonga_dir="${top_dir}" + . "${top_dir}/config.sh" +fi + +n_processors="$(grep '^processor' /proc/cpuinfo | wc -l)" +if [ "${MROONGA_BUNDLED}" = "yes" ]; then + max_n_processors=2 +else + max_n_processors=4 +fi +if (( $n_processors > $max_n_processors )); then + n_processors=$max_n_processors +fi + +build() +{ + make -j${n_processors} > /dev/null +} + +run_unit_test() +{ + if [ "${MROONGA_BUNDLED}" != "yes" ]; then + NO_MAKE=yes ${mroonga_dir}/test/run-unit-test.sh + fi +} + +prepare_mysql_test_dir() +{ + mysql_test_dir=/usr/mysql-test + if [ -d /usr/lib/mysql-testsuite/ ]; then + sudo cp -a /usr/lib/mysql-testsuite/ ${mysql_test_dir}/ + elif [ -d /usr/lib/mysql-test/ ]; then + sudo cp -a /usr/lib/mysql-test/ ${mysql_test_dir}/ + elif [ -d /usr/share/mysql/mysql-test/ ]; then + sudo cp -a /usr/share/mysql/mysql-test/ ${mysql_test_dir}/ + elif [ -d /usr/share/mysql-test/ ]; then + sudo cp -a /usr/share/mysql-test/ ${mysql_test_dir}/ + elif [ -d /opt/mysql/ ]; then + mysql_test_dir=$(echo /opt/mysql/server-*/mysql-test) + else + sudo cp -a ${MYSQL_SOURCE_DIR}/mysql-test/ ${mysql_test_dir}/ + fi + sudo chown -R $(id -u):$(id -g) ${mysql_test_dir}/ + + cp -a ${mroonga_dir}/mysql-test/mroonga/ ${mysql_test_dir}/suite/ +} + +collect_test_suite_names() +{ + cd ${mysql_test_dir}/suite/ + test_suite_names="" + for test_suite_name in $(find mroonga -type d '!' -name '[tr]'); do + if [ -n "${test_suite_names}" ]; then + test_suite_names="${test_suite_names}," + fi + test_suite_names="${test_suite_names}${test_suite_name}" + done + cd - +} + +prepare_sql_test() +{ + sudo make install > /dev/null + prepare_mysql_test_dir + collect_test_suite_names +} + +run_sql_test() +{ + test_args=() + if [ "${MROONGA_TEST_EMBEDDED}" = "yes" ]; then + test_args=("${test_args[@]}" "--embedded-server") + fi + if [ "${MROONGA_TEST_PS_PROTOCOL}" = "yes" ]; then + test_args=("${test_args[@]}" "--ps-protocol") + fi + + if [ "${MROONGA_BUNDLED}" = "yes" ]; then + # Plugins aren't supported. + cd ${mroonga_dir}/mysql-test/mroonga/storage + rm -rf alter_table/add_index/token_filters/ + rm -rf alter_table/t/change_token_filter.test + rm -rf create/table/token_filters/ + rm -rf fulltext/token_filters/ + cd - + + ${mroonga_dir}/test/run-sql-test.sh \ + "${test_args[@]}" \ + --parallel="${n_processors}" \ + --retry=3 + else + prepare_sql_test + + cd ${mysql_test_dir}/ + perl ./mysql-test-run.pl \ + "${test_args[@]}" \ + --no-check-testcases \ + --parallel="${n_processors}" \ + --retry=3 \ + --suite="${test_suite_names}" \ + --force + fi +} + +build +# run_unit_test +run_sql_test diff --git a/storage/mroonga/tools/upload-to-github.rb b/storage/mroonga/tools/upload-to-github.rb new file mode 100755 index 00000000..572d65c3 --- /dev/null +++ b/storage/mroonga/tools/upload-to-github.rb @@ -0,0 +1,42 @@ +#!/usr/bin/env ruby + +if ARGV.size < 1 + puts "Usage: #{$0} USER FILE ..." + puts " e.g.: #{$0} kou mroonga-1.10.tar.gz ..." + exit false +end + +require "rubygems" +require "github_api" +require "mime/types" + +user, *files = *ARGV + +print "password[#{user}]: " +system("stty -echo") +password = STDIN.gets.chomp +system("stty echo") +puts + +github = Github.new(:login => user, :password => password) +files.each do |file| + content_type = MIME::Types.type_for(file)[0].to_s + resource = github.repos.downloads.create("mroonga", "mroonga", + :name => File.basename(file), + :size => File.size(file), + :description => File.basename(file), + :content_type => content_type) + p resource + + system("curl", + "-F", "key=#{resource.path}", + "-F", "acl=#{resource.acl}", + "-F", "success_action_status=201", + "-F", "Filename=#{resource.name}", + "-F", "AWSAccessKeyId=#{resource.accesskeyid}", + "-F", "Policy=#{resource.policy}", + "-F", "Signature=#{resource.signature}", + "-F", "Content-Type=#{resource.mime_type}", + "-F", "file=@#{file}", + resource.s3_url) +end |