From a175314c3e5827eb193872241446f2f8f5c9d33c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 4 May 2024 20:07:14 +0200 Subject: Adding upstream version 1:10.5.12. Signed-off-by: Daniel Baumann --- storage/mroonga/vendor/groonga/tools/Makefile.am | 7 + .../groonga/tools/check-small-index-limit.rb | 123 ++++++++++++++ .../groonga/tools/groonga-benchmark-indexing.rb | 129 +++++++++++++++ .../groonga/tools/groonga-memory-leak-checker.rb | 93 +++++++++++ .../groonga/tools/groonga-memory-usage-analyzer.rb | 127 ++++++++++++++ .../groonga/tools/groonga-object-list-checker.rb | 104 ++++++++++++ .../groonga/tools/groonga-suggest-httpd-client.rb | 181 ++++++++++++++++++++ .../tools/install/install-for-debian-jessie.sh | 17 ++ .../vendor/groonga/tools/prepare-sphinx-html.rb | 183 +++++++++++++++++++++ .../vendor/groonga/tools/travis-before-script.sh | 53 ++++++ .../mroonga/vendor/groonga/tools/travis-install.sh | 38 +++++ .../mroonga/vendor/groonga/tools/travis-script.sh | 75 +++++++++ 12 files changed, 1130 insertions(+) create mode 100644 storage/mroonga/vendor/groonga/tools/Makefile.am create mode 100755 storage/mroonga/vendor/groonga/tools/check-small-index-limit.rb create mode 100755 storage/mroonga/vendor/groonga/tools/groonga-benchmark-indexing.rb create mode 100755 storage/mroonga/vendor/groonga/tools/groonga-memory-leak-checker.rb create mode 100755 storage/mroonga/vendor/groonga/tools/groonga-memory-usage-analyzer.rb create mode 100755 storage/mroonga/vendor/groonga/tools/groonga-object-list-checker.rb create mode 100755 storage/mroonga/vendor/groonga/tools/groonga-suggest-httpd-client.rb create mode 100755 storage/mroonga/vendor/groonga/tools/install/install-for-debian-jessie.sh create mode 100755 storage/mroonga/vendor/groonga/tools/prepare-sphinx-html.rb create mode 100755 storage/mroonga/vendor/groonga/tools/travis-before-script.sh create mode 100755 storage/mroonga/vendor/groonga/tools/travis-install.sh create mode 100755 storage/mroonga/vendor/groonga/tools/travis-script.sh (limited to 'storage/mroonga/vendor/groonga/tools') diff --git a/storage/mroonga/vendor/groonga/tools/Makefile.am b/storage/mroonga/vendor/groonga/tools/Makefile.am new file mode 100644 index 00000000..6027eee1 --- /dev/null +++ b/storage/mroonga/vendor/groonga/tools/Makefile.am @@ -0,0 +1,7 @@ +noinstall_ruby_scripts = \ + groonga-memory-leak-checker.rb \ + groonga-object-list-checker.rb \ + prepare-sphinx-html.rb + +EXTRA_DIST = \ + $(noinstall_ruby_scripts) diff --git a/storage/mroonga/vendor/groonga/tools/check-small-index-limit.rb b/storage/mroonga/vendor/groonga/tools/check-small-index-limit.rb new file mode 100755 index 00000000..d943d89e --- /dev/null +++ b/storage/mroonga/vendor/groonga/tools/check-small-index-limit.rb @@ -0,0 +1,123 @@ +#!/usr/bin/env ruby + +# Groonga: 70dc95ef3b6fed1225981d099a65dcb7297248c5 +# +# N segments N chunks N patterns N records +# 1 1 2 50 +# 2 2 2 18898 +# 4 4 2 31181 +# 8 8 2 57853 +# 16 16 2 91349 +# 32 32 2 178502 +# 64 64 2 475020 +# 128 128 2 1066081 +# 256 256 2 2250389 +# 512 512 2 4648072 +# nil nil 1 16779239 +# nil nil 2 4648063 +# nil nil 4 7239005 +# nil nil 8 8308626 +# nil nil 16 11068608 +# nil nil 32 12670806 +# nil nil 64 18524231 +# nil nil 128 38095525 +# nil nil 256 51265415 + +require "fileutils" +require "json" + +def check_max_index(options) + max_n_segments = options[:max_n_segments] + max_n_chunks = options[:max_n_chunks] + n_patterns = options[:n_patterns] || 2 + + ENV["GRN_II_MAX_N_SEGMENTS_SMALL"] = max_n_segments&.to_s + ENV["GRN_II_MAX_N_CHUNKS_SMALL"] = max_n_chunks&.to_s + + db_dir = "/dev/shm/db" + log_path = "#{db_dir}/log" + FileUtils.rm_rf(db_dir) + FileUtils.mkdir_p(db_dir) + command_line = [ + "groonga", + "--log-path", log_path, + "-n", "#{db_dir}/db", + ] + IO.popen(command_line, "r+") do |groonga| + groonga.puts("table_create x TABLE_HASH_KEY UInt32") + groonga.gets + groonga.puts("column_create x y COLUMN_SCALAR UInt32") + groonga.gets + groonga.puts("table_create a TABLE_PAT_KEY UInt32") + groonga.gets + groonga.puts("column_create a b COLUMN_INDEX|INDEX_SMALL x y") + groonga.gets + + groonga.puts("load --table x") + groonga.puts("[") + File.open(log_path) do |log| + log.seek(0, IO::SEEK_END) + log_size = log.size + i = 0 + catch do |abort| + loop do + y = i + 1 + n_patterns.times do + groonga.print(JSON.generate({"_key" => i, "y" => y})) + groonga.puts(",") + groonga.flush + i += 1 + if log.size != log_size + data = log.read + if /\|[Ae]\|/ =~ data + parameters = [ + max_n_segments.inspect, + max_n_chunks.inspect, + n_patterns.inspect, + i, + ] + puts(parameters.join("\t")) + # puts(data) + throw(abort) + end + log_size = log.size + end + end + end + end + end + groonga.puts("]") + load_response = groonga.gets + # puts(load_response) + + groonga.puts("quit") + groonga.gets + end +end + +puts("N segments\tN chunks\tN patterns\tN records") +[ + [1, 1, 2], + [2, 2, 2], + [4, 4, 2], + [8, 8, 2], + [16, 16, 2], + [32, 32, 2], + [64, 64, 2], + [128, 128, 2], + [256, 256, 2], + [512, 512, 2], + [nil, nil, 1], + [nil, nil, 2], + [nil, nil, 4], + [nil, nil, 8], + [nil, nil, 16], + [nil, nil, 32], + [nil, nil, 64], + [nil, nil, 128], + [nil, nil, 256], +].each do |max_n_segments, max_n_chunks, n_parameters| + check_max_index(:max_n_segments => max_n_segments, + :max_n_chunks => max_n_chunks, + :n_patterns => n_parameters) +end diff --git a/storage/mroonga/vendor/groonga/tools/groonga-benchmark-indexing.rb b/storage/mroonga/vendor/groonga/tools/groonga-benchmark-indexing.rb new file mode 100755 index 00000000..9c64e6d9 --- /dev/null +++ b/storage/mroonga/vendor/groonga/tools/groonga-benchmark-indexing.rb @@ -0,0 +1,129 @@ +#!/usr/bin/env ruby + +require "fileutils" +require "json" +require "optparse" + +class IndexingBenchmarker + def initialize + @groonga = "groonga" + @database_path = nil + @benchmark_database_dir = detect_benchmark_database_dir + end + + def run + catch(:run) do + parse_options! + end + + dump_no_indexes = dump("dump-no-indexes.grn", + "--dump_indexes", "no") + dump_only_indexes = dump("dump-only-indexes.grn", + "--dump_plugins", "no", + "--dump_schema", "no", + "--dump_records", "no", + "--dump_configs", "no") + dump_no_records = dump("dump-no-records.grn", + "--dump_records", "no") + dump_only_records = dump("dump-only-records.grn", + "--dump_plugins", "no", + "--dump_schema", "no", + "--dump_indexes", "no", + "--dump_configs", "no") + + create_benchmark_database do + p [:load_record, measure(dump_no_indexes)] + p [:static_index_creation, measure(dump_only_indexes)] + end + + create_benchmark_database do + p [:create_schema, measure(dump_no_records)] + p [:load_record_and_create_index, measure(dump_only_records)] + end + + true + end + + private + def detect_benchmark_database_dir + candiates = [ + "/dev/shm", + "tmp", + ] + candiates.find do |candidate| + File.exist?(candidate) + end + end + + def benchmark_database_path + "#{@benchmark_database_dir}/bench-db/db" + end + + def parse_options! + option_parser = OptionParser.new do |parser| + parser.banner += " SOURCE_DATABASE" + + parser.on("--groonga=PATH", + "Use PATH as groonga command path") do |path| + @groonga = path + end + + parser.on("--benchmark-database-dir=DIR", + "Use DIR to put benchmark database") do |dir| + @benchmark_database_dir = dir + end + end + + @database_path, = option_parser.parse!(ARGV) + if @database_path.nil? + puts(option_parser) + throw(:run) + end + end + + def dump(path, *dump_options) + return path if File.exist?(path) + unless system(@groonga, + @database_path, + "dump", + *dump_options, + :out => path) + raise "failed to dump: #{dump_options.inspect}" + end + path + end + + def create_benchmark_database + dir = File.dirname(benchmark_database_path) + FileUtils.rm_rf(dir) + FileUtils.mkdir_p(dir) + system(@groonga, + "-n", benchmark_database_path, + "shutdown", + :out => IO::NULL) + begin + yield + ensure + FileUtils.rm_rf(dir) + end + end + + def measure(dump_path) + result = "result" + begin + system(@groonga, + "--file", dump_path, + benchmark_database_path, + :out => result) + File.open(result) do |output| + output.each_line.inject(0) do |result, line| + result + JSON.parse(line)[0][2] + end + end + ensure + FileUtils.rm_f(result) + end + end +end + +exit(IndexingBenchmarker.new.run) diff --git a/storage/mroonga/vendor/groonga/tools/groonga-memory-leak-checker.rb b/storage/mroonga/vendor/groonga/tools/groonga-memory-leak-checker.rb new file mode 100755 index 00000000..4e4e10a4 --- /dev/null +++ b/storage/mroonga/vendor/groonga/tools/groonga-memory-leak-checker.rb @@ -0,0 +1,93 @@ +#!/usr/bin/env ruby + +unless respond_to?(:spawn, true) + puts("Ruby 1.9 is required.") + exit(false) +end + +require 'ostruct' +require 'optparse' +require 'tempfile' +require 'time' + +options = OpenStruct.new +options.groonga = "groonga" +options.show_result = false +options.report_progress = true + +option_parser = OptionParser.new do |parser| + parser.banner += " DATABASE COMMAND_FILE1 ..." + + parser.on("--groonga=PATH", + "Use PATH as groonga command path") do |path| + options.groonga = path + end + + parser.on("--[no-]show-result", + "Show result of command") do |boolean| + options.show_result = boolean + end + + parser.on("--[no-]report-progress", + "Report progress") do |boolean| + options.report_progress = boolean + end +end + +database, *command_files = option_parser.parse!(ARGV) +if database.nil? + puts(option_parser) + exit(false) +end + +i = 0 +command_files.each do |path| + File.open(path) do |file| + file.each_line do |command| + if options.report_progress + i += 1 + puts("#{Time.now.iso8601}: #{i} commands done.") if (i % 1000).zero? + end + command = command.chomp + base_name = File.basename($0, ".*") + log = Tempfile.new("#{base_name}-log") + command_file = Tempfile.new("#{base_name}-command") + command_file.puts(command) + command_file.close + command_line = [options.groonga, + "--log-path", log.path, + "--file", command_file.path] + command_file << "-n" unless File.exist?(database) + command_line << database + result = Tempfile.new("#{base_name}-result") + pid = spawn(*command_line, :out => result.fileno) + pid, status = Process.waitpid2(pid) + unless status.success? + puts("failed to run: (#{status.exitstatus}): " + + "[#{command_line.join(' ')}]") + puts("command:") + puts(command) + puts("result:") + result.open + puts(result.read) + next + # exit(false) + end + if options.show_result + result.open + puts(result.read) + end + log.open + log.each_line do |log_line| + case log_line + when /grn_fin \((\d+)\)/ + n_unfreed_allocations = $1.to_i + unless n_unfreed_allocations.zero? + puts("maybe memory leak: #{n_unfreed_allocations}: <#{command}>") + exit(false) + end + end + end + end + end +end diff --git a/storage/mroonga/vendor/groonga/tools/groonga-memory-usage-analyzer.rb b/storage/mroonga/vendor/groonga/tools/groonga-memory-usage-analyzer.rb new file mode 100755 index 00000000..5c4d5669 --- /dev/null +++ b/storage/mroonga/vendor/groonga/tools/groonga-memory-usage-analyzer.rb @@ -0,0 +1,127 @@ +#!/usr/bin/env ruby + +class Memory < Struct.new(:size, :file, :line, :function) + def location + "#{file}:#{line}" + end +end + +class LocationGroup + attr_reader :location + attr_reader :memories + def initialize(location) + @location = location + @memories = [] + end + + def add(memory) + @memories << memory + end + + def total_size + @memories.inject(0) do |sum, memory| + sum + memory.size + end + end + + def average_size + total_size / @memories.size.to_f + end + + def max_size + @memories.collect(&:size).max + end + + def min_size + @memories.collect(&:size).min + end +end + +class Statistics + def initialize + @location_groups = {} + end + + def add(memory) + group = location_group(memory.location) + group.add(memory) + end + + def sort_by_size + @location_groups.values.sort_by do |group| + group.total_size + end + end + + private + def location_group(location) + @location_groups[location] ||= LocationGroup.new(location) + end +end + +statistics = Statistics.new + +ARGF.each_line do |line| + case line + when /\Aaddress\[\d+\]\[not-freed\]:\s + (?:0x)?[\da-fA-F]+\((\d+)\):\s + (.+?):(\d+):\s(\S+)/x + size = $1.to_i + file = $2 + line = $3.to_i + function = $4.strip + memory = Memory.new(size, file, line, function) + statistics.add(memory) + end +end + +def format_size(size) + if size < 1024 + "#{size}B" + elsif size < (1024 * 1024) + "%.3fKiB" % (size / 1024.0) + elsif size < (1024 * 1024 * 1024) + "%.3fMiB" % (size / 1024.0 / 1024.0) + elsif size < (1024 * 1024 * 1024 * 1024) + "%.3fGiB" % (size / 1024.0 / 1024.0 / 1024.0) + else + "#{size}B" + end +end + +puts("%10s(%10s:%10s:%10s): %s(%s)" % [ + "Total", + "Average", + "Max", + "Min", + "Location", + "N allocations", + ]) +top_allocated_groups = statistics.sort_by_size.reverse_each.take(10) +top_allocated_groups.each do |group| + puts("%10s(%10s:%10s:%10s): %s(%d)" % [ + format_size(group.total_size), + format_size(group.average_size), + format_size(group.max_size), + format_size(group.min_size), + group.location, + group.memories.size, + ]) +end + +puts +puts("Top allocated location's details") +top_allocated_group = top_allocated_groups.first +target_memories = top_allocated_group.memories +size_width = Math.log10(target_memories.size).floor + 1 +target_memories.group_by(&:size).sort_by do |size, memories| + size * memories.size +end.reverse_each do |size, memories| + total_size = memories.inject(0) {|sum, memory| sum + memory.size} + puts("%10s(%10s * %#{size_width}d): %s" % [ + format_size(total_size), + format_size(size), + memories.size, + memories.first.location, + ]) +end diff --git a/storage/mroonga/vendor/groonga/tools/groonga-object-list-checker.rb b/storage/mroonga/vendor/groonga/tools/groonga-object-list-checker.rb new file mode 100755 index 00000000..f92eec0e --- /dev/null +++ b/storage/mroonga/vendor/groonga/tools/groonga-object-list-checker.rb @@ -0,0 +1,104 @@ +#!/usr/bin/env ruby +# +# Copyright(C) 2016 Kouhei Sutou +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +require "pp" +require "json" +require "groonga/command/parser" + +if ARGV.empty? + puts("Usage:") + puts(" #{$0} schema.grn object_list_result.json") + puts(" #{$0} schema.grn < object_list_result.json") + puts(" groonga DB_PATH object_list | #{$0} schema.grn") + exit(false) +end + +schema_grn = ARGV.shift + +schema = {} +Groonga::Command::Parser.parse(File.read(schema_grn)) do |type, *args| + case type + when :on_command + command, = args + case command.name + when "table_create" + if command.table_no_key? + type = "table:no_key" + elsif command.table_pat_key? + type = "table:pat_key" + elsif command.table_dat_key? + type = "table:dat_key" + else + type = "table:hash_key" + end + name = command[:name] + schema[name] ||= [] + schema[name] << { + :type => type, + :flags => command.flags.join("|"), + } + when "column_create" + if command.column_index? + type = "column:index" + elsif command.column_vector? or command.type == "ShortText" + type = "column:var_size" + else + type = "column:fix_size" + end + name = "#{command[:table]}.#{command[:name]}" + schema[name] ||= [] + schema[name] << { + :type => type, + :flags => command.flags.join("|"), + } + end + end +end + +MAX_RESERVED_ID = 255 +response = JSON.parse(ARGF.read) +body = response[1] +body.each do |name, object| + id = object["id"] + next if id <= MAX_RESERVED_ID + normalized_name = name.gsub(/\d{4}\d{2}(?:\d{2})?/, "YYYYMMDD") + + definitions = schema[normalized_name] + if definitions.nil? + next if object["type"]["name"] == "proc" + puts("Unknown table/column: #{name}(#{id})") + exit(false) + end + + type = object["type"] + if type.nil? + puts("[invalid][no-type] #{id}:#{name}") + puts(PP.pp(object, "").gsub(/^/, " ")) + next + end + + type_name = type["name"] + valid_type_names = definitions.collect {|definition| definition[:type]} + unless valid_type_names.include?(type["name"]) + expected = "expected:[#{valid_type_names.join(", ")}]" + puts("[invalid][wrong-type] #{id}:#{name} <#{type_name}> #{expected}") + puts(PP.pp(object, "").gsub(/^/, " ")) + puts(PP.pp(definitions, "").gsub(/^/, " ")) + next + end +end diff --git a/storage/mroonga/vendor/groonga/tools/groonga-suggest-httpd-client.rb b/storage/mroonga/vendor/groonga/tools/groonga-suggest-httpd-client.rb new file mode 100755 index 00000000..00b6abf6 --- /dev/null +++ b/storage/mroonga/vendor/groonga/tools/groonga-suggest-httpd-client.rb @@ -0,0 +1,181 @@ +#!/usr/bin/env ruby +# +# Copyright(C) 2011 Kouhei Sutou +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License version 2.1 as published by the Free Software Foundation. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +require "optparse" +require "cool.io" +require "digest" + +class Terms + def initialize + @sources = [] + end + + def next + until @sources.empty? + source = @sources.first + term = source.next + @sources.shift if source.empty? + break if term + end + term + end + + def empty? + @sources.all?(&:empty?) + end + + def add_source_file(path) + @sources << InputSource.new(File.open(path)) + end + + def add_source_input(input) + @sources << InputSource.new(input) + end + + class InputSource + def initialize(source) + @source = source + @lines = @source.each_line + @look_ahead_term = nil + end + + def next + if @look_ahead_term + term = @look_ahead_term + @look_ahead_term = nil + else + loop do + term = @lines.next.strip + break unless term.empty? + end + end + term + rescue StopIteration + @lines = nil + @source.close + @source = nil + nil + end + + def empty? + if @source.nil? + true + else + @look_ahead_term ||= self.next + @look_ahead_term.nil? + end + end + end +end + +class GroongaSuggestHTTPDClient < Coolio::HttpClient + def initialize(socket, dataset, id, term) + super(socket) + @dataset = dataset + @id = id + @term = term + @callbacks = [] + end + + def request + query = {} + query["q"] = @term.dup.force_encoding("ASCII-8BIT") + query["s"] = (Time.now.to_f * 1_000).round.to_s + query["i"] = @id + query["t"] = "submit" if rand(10).zero? + query["l"] = @dataset + super("GET", "/", :query => query) + end + + def on_body_data(data) + end + + def on_request_complete(&block) + if block + @callbacks << block + else + @callbacks.each do |callback| + callback.call(self) + end + end + end + + def on_error(reason) + close + $stderr.puts("Error: #{reason}") + end +end + +n_connections = 100 +host = "localhost" +port = 8080 +terms = Terms.new +parser = OptionParser.new +parser.banner += " DATASET" +parser.on("--host=HOST", + "Use HOST as groonga suggest HTTPD host.", + "(#{host})") do |_host| + host = _host +end +parser.on("--port=PORT", Integer, + "Use PORT as groonga suggest HTTPD port.", + "(#{port})") do |_port| + port = _port +end +parser.on("--n-connections=N", Integer, + "Use N connections.", + "(#{n_connections})") do |n| + n_connections = n +end +parser.on("--terms=PATH", + "Use terns in PATH.", + "(none)") do |path| + terms.add_source_file(path) +end + +parser.parse! +if ARGV.size != 1 + puts(parser) + exit(false) +end +dataset = ARGV.shift + +if terms.empty? and !$stdin.tty? + terms.add_source_input($stdin) +end + +if terms.empty? + puts("no terms") + exit(false) +end + +loop = Coolio::Loop.default +run_client = lambda do |id| + term = terms.next + return if term.nil? + client = GroongaSuggestHTTPDClient.connect(host, port, dataset, id, term) + client.on_request_complete do + run_client.call(id) + end + client.attach(loop) + client.request +end +n_connections.times do |i| + id = Digest::SHA2.hexdigest(Time.now.to_f.to_s) + run_client.call(id) +end +loop.run diff --git a/storage/mroonga/vendor/groonga/tools/install/install-for-debian-jessie.sh b/storage/mroonga/vendor/groonga/tools/install/install-for-debian-jessie.sh new file mode 100755 index 00000000..d053d2d9 --- /dev/null +++ b/storage/mroonga/vendor/groonga/tools/install/install-for-debian-jessie.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +sources_list_path=/etc/apt/sources.list.d/groonga.list + +if [ ! -f $sources_list_path ]; then + sudo cat <(ソースコードを表示)#{label}（英語）<" + end +end + +def fix_js_link(js, language) + fix_link_path(js) +end + +LANGUAGE_TO_LOCALE = { + "ja" => "ja_JP", + "en" => "en_US", +} + +def insert_facebook_html_header(html) + html.gsub(/<\/head>/) do + <<-HTML + + + + + + + + + HTML + end +end + +def insert_facebook_html_fb_root(html) + html.gsub(//) do + <<-HTML + +
+ HTML + end +end + +def insert_facebook_html_buttons(html) + html.gsub(/(
)/) do + <<-HTML + + #{$1} + HTML + end +end + +def insert_facebook_html_footer(html, language) + locale = LANGUAGE_TO_LOCALE[language] + raise "unknown locale for language #{language.inspect}" if locale.nil? + html.gsub(/<\/body>/) do + <<-HTML + + + + + HTML + end +end + +def insert_facebook_html(html, language) + html = insert_facebook_html_header(html) + html = insert_facebook_html_fb_root(html) + html = insert_facebook_html_buttons(html) + html = insert_facebook_html_footer(html, language) + html +end + +source_dir, dest_dir = ARGV + +source_dir = Pathname.new(source_dir) +dest_dir = Pathname.new(dest_dir) + +language_dirs = [] +source_dir.each_entry do |top_level_path| + language_dirs << top_level_path if /\A[a-z]{2}\z/ =~ top_level_path.to_s +end + +language_dirs.each do |language_dir| + language = language_dir.to_s + language_source_dir = source_dir + language_dir + "html" + language_dest_dir = dest_dir + language_dir + language_source_dir.find do |source_path| + relative_path = source_path.relative_path_from(language_source_dir) + dest_path = language_dest_dir + relative_path + if source_path.directory? + dest_path.mkpath + else + case source_path.extname + when ".html", ".js" + content = source_path.read + extension = source_path.extname.gsub(/\A\./, '') + content = fix_link(content, extension, language) + if extension == "html" + content = insert_facebook_html(content, language) + content = add_language_annotation_to_source_label(content, language) + end + dest_path.open("wb") do |dest| + dest.print(content.strip) + end + FileUtils.touch(dest_path, :mtime => source_path.mtime) + else + case source_path.basename.to_s + when ".buildinfo" + # ignore + else + FileUtils.cp(source_path, dest_path, :preserve => true) + end + end + end + end +end + +dest_dir.find do |dest_path| + if dest_path.directory? and /\A_/ =~ dest_path.basename.to_s + normalized_dest_path = dest_path + ".." + normalized_dest_path += dest_path.basename.to_s.gsub(/\A_/, '') + FileUtils.mv(dest_path, normalized_dest_path) + end +end diff --git a/storage/mroonga/vendor/groonga/tools/travis-before-script.sh b/storage/mroonga/vendor/groonga/tools/travis-before-script.sh new file mode 100755 index 00000000..87ed5756 --- /dev/null +++ b/storage/mroonga/vendor/groonga/tools/travis-before-script.sh @@ -0,0 +1,53 @@ +#!/bin/sh + +set -e +set -u + +: ${ENABLE_MRUBY:=no} + +git submodule update --init --depth 1 + +prefix=/tmp/local + +case "${BUILD_TOOL}" in + autotools) + ./autogen.sh + + configure_args="" + if [ "${TRAVIS_OS_NAME}" = "osx" ]; then + pkg_config_path="$(brew --prefix openssl)/lib/pkgconfig" + configure_args="${configure_args} PKG_CONFIG_PATH=${pkg_config_path}" + fi + #if [ "$CC" = "clang" ]; then + configure_args="${configure_args} --enable-debug" + #fi + if [ "${ENABLE_MRUBY}" = "yes" ]; then + configure_args="${configure_args} --with-ruby --enable-mruby" + fi + + ./configure --prefix=${prefix} --with-ruby ${configure_args} + ;; + cmake) + cmake_args="" + cmake_args="${cmake_args} -DGRN_WITH_DEBUG=yes" + if [ "${ENABLE_MRUBY}" = "yes" ]; then + cmake_args="${cmake_args} -DGRN_WITH_MRUBY=yes" + fi + + cmake . ${cmake_args} + ;; +esac + +case "$(uname)" in + Linux) + n_processors="$(grep '^processor' /proc/cpuinfo | wc -l)" + ;; + Darwin) + n_processors="$(/usr/sbin/sysctl -n hw.ncpu)" + ;; + *) + n_processors="1" + ;; +esac + +make -j${n_processors} > /dev/null diff --git a/storage/mroonga/vendor/groonga/tools/travis-install.sh b/storage/mroonga/vendor/groonga/tools/travis-install.sh new file mode 100755 index 00000000..d7ac400c --- /dev/null +++ b/storage/mroonga/vendor/groonga/tools/travis-install.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +set -e +set -u + +: ${ENABLE_MRUBY:=no} + +case "${TRAVIS_OS_NAME}" in + linux) + curl --silent --location https://raw.github.com/clear-code/cutter/master/data/travis/setup.sh | sh + sudo apt-get install -qq -y \ + autotools-dev \ + autoconf-archive \ + zlib1g-dev \ + libmsgpack-dev \ + libevent-dev \ + libmecab-dev \ + mecab-naist-jdic \ + cmake + ;; + osx) + brew update > /dev/null + brew outdated pkg-config || brew upgrade pkg-config + brew reinstall libtool + brew outdated libevent || brew upgrade libevent + brew install \ + autoconf-archive \ + msgpack \ + mecab \ + mecab-ipadic + brew install --force openssl + # brew install cutter + ;; +esac + +if [ "${ENABLE_MRUBY}" = "yes" ]; then + gem install pkg-config groonga-client test-unit +fi diff --git a/storage/mroonga/vendor/groonga/tools/travis-script.sh b/storage/mroonga/vendor/groonga/tools/travis-script.sh new file mode 100755 index 00000000..cc045725 --- /dev/null +++ b/storage/mroonga/vendor/groonga/tools/travis-script.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +set -e +set -u + +: ${ENABLE_MRUBY:=no} +: ${TEST_TARGET:=all} + +prefix=/tmp/local + +command_test_options="--reporter=mark --timeout=60" + +set -x + +export COLUMNS=79 + +retry() +{ + local i=0 + while ! "$@"; do + if [ $i -eq 3 ]; then + exit 1 + fi + i=$((i + 1)) + done +} + +if [ "${TRAVIS_OS_NAME}" = "osx" ]; then + memory_fs_size=$[768 * 1024 * 1024] # 768MiB + byte_per_sector=512 + n_sectors=$[${memory_fs_size} / ${byte_per_sector}] + memory_fs_device_path=$(hdid -nomount ram://${n_sectors}) + newfs_hfs ${memory_fs_device_path} + mkdir -p tmp + mount -t hfs ${memory_fs_device_path} tmp + + command_test_options="${command_test_options} --n-workers=2" +else + command_test_options="${command_test_options} --n-workers=4" +fi + +case "${BUILD_TOOL}" in + autotools) + case "${TEST_TARGET}" in + command) + test/command/run-test.sh ${command_test_options} + ;; + command-http) + retry test/command/run-test.sh ${command_test_options} \ + --interface http + ;; + command-httpd) + mkdir -p ${prefix}/var/log/groonga/httpd + retry test/command/run-test.sh ${command_test_options} \ + --testee groonga-httpd + ;; + *) + test/unit/run-test.sh -v v + test/command/run-test.sh ${command_test_options} + if [ "${ENABLE_MRUBY}" = "yes" ]; then + test/mruby/run-test.rb + test/command_line/run-test.rb + fi + retry test/command/run-test.sh ${command_test_options} \ + --interface http + mkdir -p ${prefix}/var/log/groonga/httpd + retry test/command/run-test.sh ${command_test_options} \ + --testee groonga-httpd + ;; + esac + ;; + cmake) + test/command/run-test.sh ${command_test_options} + ;; +esac -- cgit v1.2.3