diff options
Diffstat (limited to 'src/arrow/dev/benchmarking/examples')
8 files changed, 477 insertions, 0 deletions
diff --git a/src/arrow/dev/benchmarking/examples/benchmark_example.json b/src/arrow/dev/benchmarking/examples/benchmark_example.json new file mode 100644 index 000000000..d6f58c286 --- /dev/null +++ b/src/arrow/dev/benchmarking/examples/benchmark_example.json @@ -0,0 +1,32 @@ +[ + { + "benchmark_name": "Benchmark 1", + "parameter_names": ["arg0", "arg1", "arg2"], + "benchmark_description": "First benchmark", + "benchmark_type": "Time", + "units": "miliseconds", + "lessisbetter": true, + "benchmark_version": "second version", + "benchmark_language": "Python" + }, + { + "benchmark_name": "Benchmark 2", + "parameter_names": ["arg0", "arg1"], + "benchmark_description": "Description 2.", + "benchmark_type": "Time", + "units": "nanoseconds", + "lessisbetter": true, + "benchmark_version": "second version", + "benchmark_language": "Python" + }, + { + "benchmark_name": "Benchmark 3", + "parameter_names": ["arg0"], + "benchmark_description": "Third benchmark", + "benchmark_type": "Memory", + "units": "kilobytes", + "lessisbetter": true, + "benchmark_version": "1", + "benchmark_language": "Python" + } +] diff --git a/src/arrow/dev/benchmarking/examples/benchmark_run_example.csv b/src/arrow/dev/benchmarking/examples/benchmark_run_example.csv new file mode 100644 index 000000000..eab208a1c --- /dev/null +++ b/src/arrow/dev/benchmarking/examples/benchmark_run_example.csv @@ -0,0 +1,6 @@ +benchmark_run_id,benchmark_name,benchmark_version,parameter_values,value,git_commit_timestamp,git_hash,val_min,val_q1,val_q3,val_max,std_dev,n_obs,run_timestamp,run_metadata,run_notes,mac_address,benchmark_language,language_implementation_version,dependencies
+,Benchmark 2,version 0,"{""arg0"": 100, ""arg1"": 5}",2.5,2019-01-31 14:31:10 -0600,8136c46d5c60fb,1,2,3,4,1.41,8,2019-02-14 02:00:00 -0600,,,08:00:2b:01:02:03,Python,CPython 2.7,"{""six"": """", ""numpy"": ""1.14"", ""other_lib"": ""1.0""}"
+,Benchmark 2,version 0,"{""arg0"": 1000, ""arg1"": 5}",5,2019-01-31 14:31:10 -0600,8136c46d5c60fb,2,4,6,8,3.14,8,2019-02-14 02:01:00 -0600,,,08:00:2b:01:02:03,Python,CPython 2.7,"{""six"": """", ""numpy"": ""1.14"", ""other_lib"": ""1.0""}"
+,Benchmark 2,version 0,"{""arg0"": 100, ""arg1"": 5}",2.5,2019-01-31 14:31:10 -0600,8136c46d5c60fb,0.5,1,3,5,3,8,2019-02-14 02:02:00 -0600,,,08:00:2b:01:02:03,Python,CPython 3.6,"{""boost"": ""1.42"", ""numpy"": ""1.15""}"
+,Benchmark 2,version 0,"{""arg0"": 1000, ""arg1"": 5}",3,2019-01-31 14:31:10 -0600,8136c46d5c60fb,2,2.5,4,4.5,1.5,8,2019-02-14 02:03:00 -0600,,,08:00:2b:01:02:03,Python,CPython 3.6,"{""boost"": ""1.42"", ""numpy"": ""1.15""}"
+,Benchmark 2,version 0,"{""arg0"": 1000, ""arg1"": 10}",3,2019-01-31 14:31:10 -0600,8136c46d5c60fb,1,2,4,5,2,8,2019-02-14 02:03:30 -0600,,,08:00:2b:01:02:03,Python,CPython 2.7,"{""six"": """", ""numpy"": ""1.15"", ""other_lib"": ""1.0""}"
diff --git a/src/arrow/dev/benchmarking/examples/benchmark_run_example.json b/src/arrow/dev/benchmarking/examples/benchmark_run_example.json new file mode 100644 index 000000000..2ded776c9 --- /dev/null +++ b/src/arrow/dev/benchmarking/examples/benchmark_run_example.json @@ -0,0 +1,97 @@ +[ + { + "benchmark_name": "Benchmark 2", + "benchmark_version": "version 0", + "parameter_values": {"arg0": 100, "arg1": 5}, + "value": 2.5, + "git_commit_timestamp": "2019-02-08 22:35:53 +0100", + "git_hash": "324d3cf198444a", + "val_min": 1, + "val_q1": 2, + "val_q3": 3, + "val_max": 4, + "std_dev": 1.41, + "n_obs": 8, + "run_timestamp": "2019-02-14 03:00:05 -0600", + "mac_address": "08:00:2b:01:02:03", + "benchmark_language": "Python", + "language_implementation_version": "CPython 2.7", + "dependencies": {"six": "", "numpy": "1.14", "other_lib": "1.0"} + }, + { + "benchmark_name": "Benchmark 2", + "benchmark_version": "version 0", + "parameter_values": {"arg0": 1000, "arg1": 5}, + "value": 5, + "git_commit_timestamp": "2019-02-08 22:35:53 +0100", + "git_hash": "324d3cf198444a", + "val_min": 2, + "val_q1": 4, + "val_q3": 6, + "val_max": 8, + "std_dev": 3.14, + "n_obs": 8, + "run_timestamp": "2019-02-14 03:00:10 -0600", + "mac_address": "08:00:2b:01:02:03", + "benchmark_language": "Python", + "language_implementation_version": "CPython 2.7", + "dependencies": {"six": "", "numpy": "1.14", "other_lib": "1.0"} + }, + { + "benchmark_name": "Benchmark 2", + "benchmark_version": "version 0", + "parameter_values": {"arg0": 100, "arg1": 5}, + "value": 2.5, + "git_commit_timestamp": "2019-02-08 22:35:53 +0100", + "git_hash": "324d3cf198444a", + "val_min": 0.5, + "val_q1": 1, + "val_q3": 3, + "val_max": 5, + "std_dev": 3, + "n_obs": 8, + "run_timestamp": "2019-02-14 03:00:20 -0600", + "mac_address": "08:00:2b:01:02:03", + "benchmark_language": "Python", + "language_implementation_version": "CPython 2.7", + "dependencies": {"boost": "1.42", "numpy": "1.15"} + }, + { + "benchmark_name": "Benchmark 2", + "benchmark_version": "version 0", + "parameter_values": {"arg0": 1000, "arg1": 5}, + "value": 3, + "git_commit_timestamp": "2019-02-08 22:35:53 +0100", + "git_hash": "324d3cf198444a", + "val_min": 2, + "val_q1": 2.5, + "val_q3": 4, + "val_max": 4.5, + "std_dev": 1.5, + "n_obs": 8, + "run_timestamp": "2019-02-14 03:00:30 -0600", + "mac_address": "08:00:2b:01:02:03", + "benchmark_language": "Python", + "language_implementation_version": "CPython 2.7", + "dependencies": {"boost": "1.42", "numpy": "1.15"} + }, + { + "benchmark_name": "Benchmark 2", + "benchmark_version": "version 0", + "parameter_values": {"arg0": 1000, "arg1": 10}, + "value": 3, + "git_commit_timestamp": "2019-02-08 22:35:53 +0100", + "git_hash": "324d3cf198444a", + "val_min": 1, + "val_q1": 2, + "val_q3": 4, + "val_max": 5, + "std_dev": 2, + "n_obs": 8, + "run_timestamp": "2019-02-14 03:00:40 -0600", + "mac_address": "08:00:2b:01:02:03", + "benchmark_language": "Python", + "language_implementation_version": "CPython 2.7", + "dependencies": {"six": "", "numpy": "1.15", "other_lib": "1.0"} + } +] diff --git a/src/arrow/dev/benchmarking/examples/benchmark_with_context_example.json b/src/arrow/dev/benchmarking/examples/benchmark_with_context_example.json new file mode 100644 index 000000000..f9e6e3130 --- /dev/null +++ b/src/arrow/dev/benchmarking/examples/benchmark_with_context_example.json @@ -0,0 +1,73 @@ +{ + "context": { + "mac_address": "08:00:2b:01:02:03", + "benchmark_language": "Python", + "language_implementation_version": "CPython 2.7", + "dependencies": {"six": "", "numpy": "1.14", "other_lib": "1.0"}, + "git_commit_timestamp": "2019-02-14 22:42:22 +0100", + "git_hash": "123456789abcde", + "run_timestamp": "2019-02-25 03:00:40 -0600", + "Extra stuff": "does not hurt anything and won't be added.", + "However": "all of the entries above 'Extra stuff' are required." + }, + "benchmark_version": { + "Benchmark 2": "version 0", + "Benchmark 3": "any string is a version. (Benchmark 3 not actually used)" + }, + "benchmarks": [ + { + "benchmark_name": "Benchmark 2", + "parameter_values": {"arg0": 1, "arg1": 5}, + "value": 2.5, + "val_min": 1, + "val_q1": 2, + "val_q3": 3, + "val_max": 4, + "std_dev": 1.41, + "n_obs": 8, + "run_metadata": {"any": "json object is admissible"}, + "run_notes": "This value is an arbitrary-length string." + }, + { + "benchmark_name": "Benchmark 2", + "parameter_values": {"arg0": 2, "arg1": 5}, + "value": 5, + "std_dev": 3.14, + "n_obs": 8 + }, + { + "benchmark_name": "Benchmark 2", + "parameter_values": {"arg0": 3, "arg1": 5}, + "value": 2.5, + "val_min": 0.5, + "val_q1": 1, + "val_q3": 3, + "val_max": 5, + "std_dev": 3, + "n_obs": 8, + "run_notes": "The previous run in this list has the minimal set of keys." + }, + { + "benchmark_name": "Benchmark 2", + "parameter_values": {"arg0": 4, "arg1": 5}, + "value": 3, + "val_min": 2, + "val_q1": 2.5, + "val_q3": 4, + "val_max": 4.5, + "std_dev": 1.5, + "n_obs": 8 + }, + { + "benchmark_name": "Benchmark 2", + "parameter_values": {"arg0": 5, "arg1": 5}, + "value": 3, + "val_min": 1, + "val_q1": 2, + "val_q3": 4, + "val_max": 5, + "std_dev": 2, + "n_obs": 8 + } + ] +} diff --git a/src/arrow/dev/benchmarking/examples/example.sql b/src/arrow/dev/benchmarking/examples/example.sql new file mode 100644 index 000000000..e93269af7 --- /dev/null +++ b/src/arrow/dev/benchmarking/examples/example.sql @@ -0,0 +1,232 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/ + + +-- Example insert into each of the views: +INSERT INTO public.project(project_name, project_url, repo_url) +VALUES ( + 'Apache Arrow' + , 'https://arrow.apache.org/' + , 'https://github.com/apache/arrow'); + +INSERT INTO public.environment_view + (benchmark_language, language_implementation_version, dependencies) +VALUES + ('Python', 'CPython 2.7', '{"six": "", "numpy": "1.14", "other_lib": "1.0"}'), + ('Python', 'CPython 3.6', '{"boost": "1.42", "numpy": "1.15"}'); + +INSERT INTO public.dependencies(dependencies) +VALUES + ('{"boost": "1.68", "numpy": "1.14"}'), + ('{"boost": "1.42", "numpy": "1.16"}'); + +INSERT INTO public.language_implementation_version_view + (benchmark_language, language_implementation_version) +VALUES + ('Python', 'CPython 2.7'), + ('Python', 'CPython 3.6'); + +INSERT INTO public.unit_view + (benchmark_type, units, lessisbetter) +VALUES + ('Memory', 'gigabytes', True), + ('Memory', 'kilobytes', True); + + +\echo 'use \\dv to list the views views'; +\dv + + +SELECT * FROM environment_view; +SELECT * FROM unit_view; + + +INSERT INTO public.machine_view ( + mac_address + , machine_name + , memory_bytes + , cpu_actual_frequency_hz + , os_name + , architecture_name + , kernel_name + , cpu_model_name + , cpu_core_count + , cpu_thread_count + , cpu_frequency_max_hz + , cpu_frequency_min_hz + , cpu_l1d_cache_bytes + , cpu_l1i_cache_bytes + , cpu_l2_cache_bytes + , cpu_l3_cache_bytes + , machine_other_attributes +) VALUES ( + '08:00:2b:01:02:03' -- mac_address + , 'My-Machine-Name' -- machine_name + , 8589934592 -- memory_bytes + -- All (?) standard mac address formats are allowable: + -- https://www.postgresql.org/docs/11/datatype-net-types.html + , 2300000000 -- cpu_actual_frequency_Hz + , 'OSX' -- os_name + , 'x86_64' -- architecture_name + , '18.2.0' -- kernel + , 'Intel(R) Core(TM) i5-7360U CPU @ 2.30GHz' -- cpu_model_name + , 2 -- cpu_core_count + , 4 -- cpu_thread_count + , 2300000000 -- cpu_frequency_max_Hz + , 2300000000 -- cpu_frequency_min_Hz + , 32768 -- cpu_l1d_cache_bytes + , 32768 -- cpu_l1i_cache_bytes + , 262144 -- cpu_l2_cache_bytes + , 4194304 -- cpu_l3_cache_bytes + , '{"example": "for machine_other_attributes"}'::jsonb +); + + +INSERT INTO public.full_benchmark_run_view ( + benchmark_name + , parameter_names + , benchmark_description + , benchmark_type + , units + , lessisbetter + , benchmark_version + -- datum + , parameter_values + , value + , git_commit_timestamp + , git_hash + , val_min + , val_q1 + , val_q3 + , val_max + , std_dev + , n_obs + , run_timestamp + , run_metadata + , run_notes + -- machine_view + , machine_name + , mac_address + , memory_bytes + , cpu_actual_frequency_hz + , os_name + , architecture_name + , kernel_name + , cpu_model_name + , cpu_core_count + , cpu_thread_count + , cpu_frequency_max_hz + , cpu_frequency_min_hz + , cpu_l1d_cache_bytes + , cpu_l1i_cache_bytes + , cpu_l2_cache_bytes + , cpu_l3_cache_bytes + , machine_other_attributes + -- environment_view + , benchmark_language + , language_implementation_version + , dependencies +) VALUES ( + 'Benchmark 3' + , '{"arg0"}'::text[] + , 'Third benchmark' + , 'Memory' + , 'kilobytes' + , TRUE + , '0' + -- datum + , '{"arg0": 10}'::jsonb + , 0.5 + , '2019-01-31 14:31:10 -0600' + , '8136c46d5c60fb' + , 0.5 + , 0.5 + , 0.5 + , 0.5 + , 0 + , 2 + , '2019-02-14 14:00:00 -0600' + , '{"ci_99": [2.7e-06, 3.1e-06]}'::jsonb + , 'Additional run_notes.' + -- machine_view + , 'My-Machine-Name' + , '09-00-2c-01-02-03' + , 8589934592 + , 2300000000 + , 'OSX' + , 'x86_64' + , '18.2.0' + , 'Intel(R) Core(TM) i5-7360U CPU @ 2.30GHz' + , 2 + , 4 + , 2300000000 + , 2300000000 + , 32768 + , 32768 + , 262144 + , 4194304 + , '{"example": "for machine_other_attributes"}'::jsonb + -- environment_view + , 'Python' + , 'CPython 2.7' + , '{"six": "", "numpy": "1.15", "other_lib": "1.0"}'::jsonb +); + + +-- Bulk load from CSV. First column is empty; serial "benchmark_run_id" will be assigned. +--\copy benchmark_run_view FROM 'examples/benchmark_run_example.csv' WITH (FORMAT csv, HEADER); + +-- Load from JSON +--\set content `cat examples/benchmark_example.json` +--SELECT ingest_benchmark_view(:'content'::jsonb); + +INSERT INTO public.benchmark_view ( + benchmark_name + , parameter_names + , benchmark_description + , benchmark_type + , units + , lessisbetter + , benchmark_version + , benchmark_language + ) VALUES ( + 'Benchmark 1' + , '{"arg0", "arg1", "arg2"}'::text[] + , E'Description.\nNewlines are OK in a string escaped with leading "E".' + , 'Time' + , 'miliseconds' + , TRUE + , 'Hash of code or other way to identify distinct benchmark versions.' + , 'Python' + ), ( + 'Benchmark 2' + , '{"arg0", "arg1"}'::text[] + , 'Description 2.' + , 'Time' + , 'nanoseconds' + , TRUE + , 'version 0' + , 'Python' + ); + + +\x +SELECT * from benchmark_run_view; + +\x diff --git a/src/arrow/dev/benchmarking/examples/example_graphql_mutation.json b/src/arrow/dev/benchmarking/examples/example_graphql_mutation.json new file mode 100644 index 000000000..fec5eed0a --- /dev/null +++ b/src/arrow/dev/benchmarking/examples/example_graphql_mutation.json @@ -0,0 +1,12 @@ +{ + "query": "mutation ($p: CreateProjectInput!){createProject(input:$p){project{id}}}", + "variables": { + "p": { + "project": { + "projectName": "Apache Arrow", + "projectUrl": "https://www.arrow.apache.org", + "repoUrl": "https://www.github.com/apache/arrow" + } + } + } +} diff --git a/src/arrow/dev/benchmarking/examples/graphql_query_environment_view.json b/src/arrow/dev/benchmarking/examples/graphql_query_environment_view.json new file mode 100644 index 000000000..78804fa91 --- /dev/null +++ b/src/arrow/dev/benchmarking/examples/graphql_query_environment_view.json @@ -0,0 +1,3 @@ +{ + "query": "{allEnvironmentViews(orderBy: [BENCHMARK_LANGUAGE_ASC, LANGUAGE_IMPLEMENTATION_VERSION_ASC, DEPENDENCIES_ASC]) {edges {node {environmentId, benchmarkLanguage, languageImplementationVersion, dependencies}}}}" +} diff --git a/src/arrow/dev/benchmarking/examples/machine.json b/src/arrow/dev/benchmarking/examples/machine.json new file mode 100644 index 000000000..2485e2bc1 --- /dev/null +++ b/src/arrow/dev/benchmarking/examples/machine.json @@ -0,0 +1,22 @@ +{ + "mac_address": "0a:00:2d:01:02:03", + "machine_name": "Yet-Another-Machine-Name", + "memory_bytes": 8589934592, + "cpu_actual_frequency_hz": 2300000000, + "os_name": "OSX", + "architecture_name": "x86_64", + "kernel_name": "18.2.0", + "cpu_model_name": "Intel(R) Core(TM) i5-7360U CPU @ 2.30GHz", + "cpu_core_count": 2, + "cpu_thread_count": 4, + "cpu_frequency_max_hz": 2300000000, + "cpu_frequency_min_hz": 2300000000, + "cpu_l1d_cache_bytes": 32768, + "cpu_l1i_cache_bytes": 32768, + "cpu_l2_cache_bytes": 262144, + "cpu_l3_cache_bytes": 4194304, + "machine_other_attributes": {"just": "an example"}, + "gpu_information": "", + "gpu_part_number": "", + "gpu_product_name": "" +} |