diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-21 11:54:28 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-21 11:54:28 +0000 |
commit | e6918187568dbd01842d8d1d2c808ce16a894239 (patch) | |
tree | 64f88b554b444a49f656b6c656111a145cbbaa28 /src/arrow/js | |
parent | Initial commit. (diff) | |
download | ceph-e6918187568dbd01842d8d1d2c808ce16a894239.tar.xz ceph-e6918187568dbd01842d8d1d2c808ce16a894239.zip |
Adding upstream version 18.2.2.upstream/18.2.2
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/arrow/js')
236 files changed, 39974 insertions, 0 deletions
diff --git a/src/arrow/js/.eslintignore b/src/arrow/js/.eslintignore new file mode 100644 index 000000000..94ef668a6 --- /dev/null +++ b/src/arrow/js/.eslintignore @@ -0,0 +1,5 @@ +.eslintrc.js +gulp +jest.config.js +jestconfigs +targets diff --git a/src/arrow/js/.eslintrc.js b/src/arrow/js/.eslintrc.js new file mode 100644 index 000000000..6d5020db1 --- /dev/null +++ b/src/arrow/js/.eslintrc.js @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + env: { + browser: true, + es6: true, + node: true, + }, + parser: "@typescript-eslint/parser", + parserOptions: { + project: "tsconfig.json", + sourceType: "module", + ecmaVersion: 2020, + }, + plugins: ["@typescript-eslint", "jest"], + extends: [ + "eslint:recommended", + "plugin:jest/recommended", + "plugin:jest/style", + "plugin:@typescript-eslint/recommended", + ], + rules: { + "@typescript-eslint/member-delimiter-style": [ + "error", + { + multiline: { + delimiter: "semi", + requireLast: true, + }, + singleline: { + delimiter: "semi", + requireLast: false, + }, + }, + ], + "@typescript-eslint/no-namespace": ["error", { "allowDeclarations": true }], + "@typescript-eslint/no-require-imports": "error", + "@typescript-eslint/no-var-requires": "off", // handled by rule above + "@typescript-eslint/quotes": [ + "error", + "single", + { + avoidEscape: true, + allowTemplateLiterals: true + }, + ], + "@typescript-eslint/semi": ["error", "always"], + "@typescript-eslint/type-annotation-spacing": "error", + "@typescript-eslint/indent": "off", + "@typescript-eslint/no-empty-function": "off", + "@typescript-eslint/no-unused-expressions": "off", + "@typescript-eslint/no-use-before-define": "off", + "@typescript-eslint/explicit-module-boundary-types": "off", + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/no-misused-new": "off", + "@typescript-eslint/ban-ts-comment": "off", + "@typescript-eslint/no-non-null-assertion": "off", + "@typescript-eslint/no-unused-vars": "off", // ts already takes care of this + + "prefer-const": ["error", { + "destructuring": "all" + }], + "curly": ["error", "multi-line"], + "brace-style": ["error", "1tbs", { "allowSingleLine": true }], + "eol-last": "error", + "no-multiple-empty-lines": "error", + "no-trailing-spaces": "error", + "no-var": "error", + "no-empty": "off", + "no-cond-assign": "off" + }, +}; diff --git a/src/arrow/js/.gitignore b/src/arrow/js/.gitignore new file mode 100644 index 000000000..799f789d6 --- /dev/null +++ b/src/arrow/js/.gitignore @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Logs +logs +*.log +.esm-cache +npm-debug.log* +yarn-debug.log* +yarn-error.log* + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (http://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ +jspm_packages/ + +# Optional npm cache directory +.npm + +# JS package manager files +package-lock.json + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env + +# compilation targets +doc +dist +targets + +# test data files +test/data/**/*.json +test/data/**/*.arrow + +# jest snapshots (too big) +test/__snapshots__/ + +# VSCode +!.vscode diff --git a/src/arrow/js/.npmrc b/src/arrow/js/.npmrc new file mode 100644 index 000000000..e55040aba --- /dev/null +++ b/src/arrow/js/.npmrc @@ -0,0 +1,7 @@ +fund=false +audit=false +save-prefix= +save-exact=true +engine-strict=true +update-notifier=false +registry=https://registry.npmjs.org/ diff --git a/src/arrow/js/.vscode/extensions.json b/src/arrow/js/.vscode/extensions.json new file mode 100644 index 000000000..1cb01b6b9 --- /dev/null +++ b/src/arrow/js/.vscode/extensions.json @@ -0,0 +1,6 @@ +{ + "recommendations": [ + "dbaeumer.vscode-eslint", + "augustocdias.tasks-shell-input", + ] +} diff --git a/src/arrow/js/.vscode/launch.json b/src/arrow/js/.vscode/launch.json new file mode 100644 index 000000000..ae72e1f48 --- /dev/null +++ b/src/arrow/js/.vscode/launch.json @@ -0,0 +1,240 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "inputs": [ + { + "type": "pickString", + "default": "src", + "id": "TEST_TARGET", + "options": [ + "src", + "apache-arrow", + "ts", + "es5.cjs", + "es5.esm", + "es5.umd", + "es2015.cjs", + "es2015.esm", + "es2015.umd", + "esnext.cjs", + "esnext.esm", + "esnext.umd", + ], + "description": "The JS version + Module format combination to test (or src to test source files)", + }, + { + "type": "command", + "id": "TEST_FILE", + "command": "shellCommand.execute", + "args": { + "cwd": "${workspaceFolder}", + "description": "Select a file to debug", + "command": "./node_modules/.bin/jest --listTests | sed -r \"s@$PWD/test/@@g\"", + } + }, + { + "type": "command", + "id": "TEST_RUNTIME_ARGS", + "command": "shellCommand.execute", + "args": { + "useSingleResult": "true", + "command": "case \"${input:TEST_TARGET}\" in *cjs | *umd | apache-arrow) echo '';; *) echo '--experimental-vm-modules';; esac" + } + }, + ], + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Debug Gulp Build", + "program": "${workspaceFolder}/node_modules/gulp/bin/gulp.js", + "args": [ + "build", + // Specify we want to debug the "src" target, which won't clean or build -- essentially a "dry-run" of the gulp build + "--target", + "src" + ] + }, + { + "type": "node", + "request": "launch", + "name": "Debug Unit Tests", + "cwd": "${workspaceRoot}", + "console": "integratedTerminal", + "program": "${workspaceFolder}/node_modules/.bin/jest", + "skipFiles": [ + "<node_internals>/**/*.js", + "${workspaceFolder}/node_modules/**/*.js" + ], + "env": { + "NODE_NO_WARNINGS": "1", + "NODE_ENV": "production", + "TEST_DOM_STREAMS": "true", + "TEST_NODE_STREAMS": "true", + }, + "runtimeArgs": ["${input:TEST_RUNTIME_ARGS}"], + "args": [ + "--verbose", + "--runInBand", + "-c", "jestconfigs/jest.${input:TEST_TARGET}.config.js", + "${input:TEST_FILE}" + ] + }, + { + "type": "node", + "request": "launch", + "name": "Debug Integration Tests", + "cwd": "${workspaceRoot}", + "program": "${workspaceFolder}/bin/integration.js", + "skipFiles": [ + "<node_internals>/**/*.js", + "${workspaceFolder}/node_modules/**/*.js" + ], + "env": { + "NODE_NO_WARNINGS": "1", + }, + "args": [ + "--mode", + "VALIDATE" + ] + }, + { + "type": "node", + "request": "launch", + "name": "Debug bin/arrow2csv", + "env": { + "ARROW_JS_DEBUG": "src", + "TS_NODE_CACHE": "false" + }, + "runtimeArgs": [ + "-r", + "ts-node/register" + ], + "console": "integratedTerminal", + "skipFiles": [ + "<node_internals>/**/*.js", + "${workspaceFolder}/node_modules/**/*.js" + ], + "args": [ + "${workspaceFolder}/src/bin/arrow2csv.ts", + "-f", + "./test/data/cpp/stream/simple.arrow" + ] + }, + { + "type": "node", + "request": "launch", + "name": "Debug bin/file-to-stream", + "env": { + "ARROW_JS_DEBUG": "src", + "TS_NODE_CACHE": "false" + }, + "runtimeArgs": [ + "-r", + "ts-node/register" + ], + "skipFiles": [ + "<node_internals>/**/*.js", + "${workspaceFolder}/node_modules/**/*.js" + ], + "args": [ + "${workspaceFolder}/bin/file-to-stream.js", + "./test/data/cpp/file/struct_example.arrow", + "./struct_example-stream-out.arrow", + ] + }, + { + "type": "node", + "request": "launch", + "name": "Debug bin/stream-to-file", + "env": { + "ARROW_JS_DEBUG": "src", + "TS_NODE_CACHE": "false" + }, + "runtimeArgs": [ + "-r", + "ts-node/register" + ], + "skipFiles": [ + "<node_internals>/**/*.js", + "${workspaceFolder}/node_modules/**/*.js" + ], + "args": [ + "${workspaceFolder}/bin/stream-to-file.js", + "./test/data/cpp/stream/struct_example.arrow", + "./struct_example-file-out.arrow", + ] + }, + { + "type": "node", + "request": "launch", + "name": "Debug bin/json-to-arrow", + "env": { + "ARROW_JS_DEBUG": "src", + "TS_NODE_CACHE": "false" + }, + "runtimeArgs": [ + "-r", + "ts-node/register" + ], + "skipFiles": [ + "<node_internals>/**/*.js", + "${workspaceFolder}/node_modules/**/*.js" + ], + "args": [ + "${workspaceFolder}/bin/json-to-arrow.js", + "-j", + "./test/data/json/struct_example.json", + "-a", + "./struct_example-stream-out.arrow", + "-f", + "stream" + ] + }, + { + "type": "node", + "request": "launch", + "name": "Debug bin/print-buffer-alignment", + "env": { + "ARROW_JS_DEBUG": "src", + "TS_NODE_CACHE": "false" + }, + "runtimeArgs": [ + "-r", + "ts-node/register" + ], + "skipFiles": [ + "<node_internals>/**/*.js", + "${workspaceFolder}/node_modules/**/*.js" + ], + "args": [ + "${workspaceFolder}/bin/print-buffer-alignment.js", + "./test/data/cpp/stream/struct_example.arrow" + ] + },{ + "type": "node", + "name": "vscode-jest-tests", + "request": "launch", + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen", + "disableOptimisticBPs": true, + "cwd": "${workspaceFolder}", + "program": "${workspaceFolder}/node_modules/.bin/jest", + "runtimeArgs": [ + "--experimental-vm-modules" + ], + "args": [ + "--runInBand", + "--watchAll=false" + ], + "env": { + "NODE_NO_WARNINGS": "1", + "TEST_DOM_STREAMS": "true", + "TEST_NODE_STREAMS": "true", + "TEST_TS_SOURCE": "true" + }, + } + ] +} diff --git a/src/arrow/js/.vscode/settings.json b/src/arrow/js/.vscode/settings.json new file mode 100644 index 000000000..379ddf14d --- /dev/null +++ b/src/arrow/js/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "typescript.tsdk": "node_modules/typescript/lib", + "editor.trimAutoWhitespace": true, + "editor.codeActionsOnSave": { + "source.fixAll.eslint": true + } +} diff --git a/src/arrow/js/DEVELOP.md b/src/arrow/js/DEVELOP.md new file mode 100644 index 000000000..66cefb084 --- /dev/null +++ b/src/arrow/js/DEVELOP.md @@ -0,0 +1,120 @@ +<!--- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +# Getting Involved + +Even if you do not plan to contribute to Apache Arrow itself or Arrow +integrations in other projects, we'd be happy to have you involved: + +* Join the mailing list: send an email to [dev-subscribe@arrow.apache.org][1]. + Share your ideas and use cases for the project. +* [Follow our activity on JIRA][3] +* [Learn the format][2] +* Contribute code to one of the reference implementations + +We prefer to receive contributions in the form of GitHub pull requests. +Please send pull requests against the [github.com/apache/arrow][4] repository. + +If you are looking for some ideas on what to contribute, check out the [JIRA +issues][3] for the Apache Arrow project. Comment on the issue and/or contact +[dev@arrow.apache.org](http://mail-archives.apache.org/mod_mbox/arrow-dev/) +with your questions and ideas. + +If you’d like to report a bug but don’t have time to fix it, you can still post +it on JIRA, or email the mailing list +[dev@arrow.apache.org](http://mail-archives.apache.org/mod_mbox/arrow-dev/) + +# The package.json scripts + +We use [yarn](https://yarnpkg.com/) to install dependencies and run scrips. + +* `yarn clean` - cleans targets +* `yarn build` - cleans and compiles all targets +* `yarn test` - executes tests against built targets + +These scripts accept argument lists of targets × modules: + +* Available `targets` are `es5`, `es2015`, `esnext`, `ts`, and `all` (default: `all`) +* Available `modules` are `cjs`, `esm`, `umd`, and `all` (default: `all`) + +Examples: + +* `yarn build` -- builds all ES targets in all module formats +* `yarn build -t es5 -m all` -- builds the ES5 target in all module formats +* `yarn build -t all -m cjs` -- builds all ES targets in the CommonJS module format +* `yarn build -t es5 -t es2015 -m all` -- builds the ES5 and ES2015 targets in all module formats +* `yarn build -t es5 -m cjs -m esm` -- builds the ES5 target in CommonJS and ESModules module formats + +This argument configuration also applies to `clean` and `test` scripts. + +To run tests on the bundles, you need to build them first. +To run tests directly on the sources without bundling, use the `src` target (e.g. `yarn test -t src`). + +* `yarn deploy` + +Uses [lerna](https://github.com/lerna/lerna) to publish each build target to npm with [conventional](https://conventionalcommits.org/) [changelogs](https://github.com/conventional-changelog/conventional-changelog/tree/master/packages/conventional-changelog-cli). + +# Running the Performance Benchmarks + +You can run the benchmarks with `yarn perf`. To print the results to stderr as JSON, add the `--json` flag (e.g. `yarn perf --json 2> perf.json`). + +You can change the target you want to test by changing the imports in `perf/index.ts`. Note that you need to compile the bundles with `yarn build` before you can import them. + +# Updating the Arrow format flatbuffers generated code + +1. Once generated, the flatbuffers format code needs to be adjusted for our build scripts (assumes `gnu-sed`): + + ```shell + cd $ARROW_HOME + + # Create a tmpdir to store modified flatbuffers schemas + tmp_format_dir=$(mktemp -d) + cp ./format/*.fbs $tmp_format_dir + + # Remove namespaces from the flatbuffers schemas + sed -i '+s+namespace org.apache.arrow.flatbuf;++ig' $tmp_format_dir/*.fbs + sed -i '+s+org.apache.arrow.flatbuf.++ig' $tmp_format_dir/*.fbs + + # Generate TS source from the modified Arrow flatbuffers schemas + flatc --ts --no-ts-reexport -o ./js/src/fb $tmp_format_dir/{File,Schema,Message}.fbs + + # Remove the tmpdir + rm -rf $tmp_format_dir + + cd ./js/src/fb + + # Rename the existing files to <filename>.bak.ts + mv File{,.bak}.ts && mv Schema{,.bak}.ts && mv Message{,.bak}.ts + + # Remove `_generated` from the ES6 imports of the generated files + sed -i '+s+_generated\";+\";+ig' *_generated.ts + # Fix all the `flatbuffers` imports + sed -i '+s+./flatbuffers+flatbuffers+ig' *_generated.ts + # Fix the Union createTypeIdsVector typings + sed -i -r '+s+static createTypeIdsVector\(builder: flatbuffers.Builder, data: number\[\] \| Uint8Array+static createTypeIdsVector\(builder: flatbuffers.Builder, data: number\[\] \| Int32Array+ig' Schema_generated.ts + # Remove "_generated" suffix from TS files + mv File{_generated,}.ts && mv Schema{_generated,}.ts && mv Message{_generated,}.ts + ``` + +2. Execute `yarn lint` from the `js` directory to fix the linting errors + +[1]: mailto:dev-subscribe@arrow.apache.org +[2]: https://github.com/apache/arrow/tree/master/format +[3]: https://issues.apache.org/jira/browse/ARROW +[4]: https://github.com/apache/arrow diff --git a/src/arrow/js/README.md b/src/arrow/js/README.md new file mode 100644 index 000000000..4ab994026 --- /dev/null +++ b/src/arrow/js/README.md @@ -0,0 +1,241 @@ +<!--- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +# [Apache Arrow](https://github.com/apache/arrow) in JS + +[![npm version](https://img.shields.io/npm/v/apache-arrow.svg)](https://www.npmjs.com/package/apache-arrow) + +Arrow is a set of technologies that enable big data systems to process and transfer data quickly. + +## Install `apache-arrow` from NPM + +`npm install apache-arrow` or `yarn add apache-arrow` + +(read about how we [package apache-arrow](#packaging) below) + +# Powering Columnar In-Memory Analytics + +[Apache Arrow](https://github.com/apache/arrow) is a columnar memory layout specification for encoding vectors and table-like containers of flat and nested data. The Arrow spec aligns columnar data in memory to minimize cache misses and take advantage of the latest SIMD (Single input multiple data) and GPU operations on modern processors. + +Apache Arrow is the emerging standard for large in-memory columnar data ([Spark](https://spark.apache.org/), [Pandas](https://wesmckinney.com/blog/pandas-and-apache-arrow/), [Drill](https://drill.apache.org/), [Graphistry](https://www.graphistry.com), ...). By standardizing on a common binary interchange format, big data systems can reduce the costs and friction associated with cross-system communication. + +# Get Started + +Check out our [API documentation][7] to learn more about how to use Apache Arrow's JS implementation. You can also learn by example by checking out some of the following resources: + +* [Observable: Introduction to Apache Arrow][5] +* [Observable: Manipulating flat arrays arrow-style][6] +* [Observable: Rich columnar data tables - Dictionary-encoded strings, 64bit ints, and nested structs][8] +* [/js/test/unit](https://github.com/apache/arrow/tree/master/js/test/unit) - Unit tests for Table and Vector + +## Cookbook + +### Get a table from an Arrow file on disk (in IPC format) + +```js +import { readFileSync } from 'fs'; +import { Table } from 'apache-arrow'; + +const arrow = readFileSync('simple.arrow'); +const table = Table.from([arrow]); + +console.log(table.toString()); + +/* + foo, bar, baz + 1, 1, aa +null, null, null + 3, null, null + 4, 4, bbb + 5, 5, cccc +*/ +``` + +### Create a Table when the Arrow file is split across buffers + +```js +import { readFileSync } from 'fs'; +import { Table } from 'apache-arrow'; + +const table = Table.from([ + 'latlong/schema.arrow', + 'latlong/records.arrow' +].map((file) => readFileSync(file))); + +console.log(table.toString()); + +/* + origin_lat, origin_lon +35.393089294433594, -97.6007308959961 +35.393089294433594, -97.6007308959961 +35.393089294433594, -97.6007308959961 +29.533695220947266, -98.46977996826172 +29.533695220947266, -98.46977996826172 +*/ +``` + +### Create a Table from JavaScript arrays + +```js +import { + Table, + FloatVector, + DateVector +} from 'apache-arrow'; + +const LENGTH = 2000; + +const rainAmounts = Float32Array.from( + { length: LENGTH }, + () => Number((Math.random() * 20).toFixed(1))); + +const rainDates = Array.from( + { length: LENGTH }, + (_, i) => new Date(Date.now() - 1000 * 60 * 60 * 24 * i)); + +const rainfall = Table.new( + [FloatVector.from(rainAmounts), DateVector.from(rainDates)], + ['precipitation', 'date'] +); +``` + +### Load data with `fetch` + +```js +import { Table } from "apache-arrow"; + +const table = await Table.from(fetch("/simple.arrow")); +console.log(table.toString()); + +``` + +### Columns look like JS Arrays + +```js +import { readFileSync } from 'fs'; +import { Table } from 'apache-arrow'; + +const table = Table.from([ + 'latlong/schema.arrow', + 'latlong/records.arrow' +].map(readFileSync)); + +const column = table.getColumn('origin_lat'); + +// Copy the data into a TypedArray +const typed = column.toArray(); +assert(typed instanceof Float32Array); + +for (let i = -1, n = column.length; ++i < n;) { + assert(column.get(i) === typed[i]); +} +``` + +# Getting involved + +See [DEVELOP.md](DEVELOP.md) + +Even if you do not plan to contribute to Apache Arrow itself or Arrow +integrations in other projects, we'd be happy to have you involved: + +* Join the mailing list: send an email to + [dev-subscribe@arrow.apache.org][1]. Share your ideas and use cases for the + project. +* [Follow our activity on JIRA][3] +* [Learn the format][2] +* Contribute code to one of the reference implementations + +We prefer to receive contributions in the form of GitHub pull requests. Please send pull requests against the [github.com/apache/arrow][4] repository. + +If you are looking for some ideas on what to contribute, check out the [JIRA +issues][3] for the Apache Arrow project. Comment on the issue and/or contact +[dev@arrow.apache.org](https://mail-archives.apache.org/mod_mbox/arrow-dev/) +with your questions and ideas. + +If you’d like to report a bug but don’t have time to fix it, you can still post +it on JIRA, or email the mailing list +[dev@arrow.apache.org](https://mail-archives.apache.org/mod_mbox/arrow-dev/) + +## Packaging + +`apache-arrow` is written in TypeScript, but the project is compiled to multiple JS versions and common module formats. + +The base `apache-arrow` package includes all the compilation targets for convenience, but if you're conscientious about your `node_modules` footprint, we got you. + +The targets are also published under the `@apache-arrow` namespace: + +```sh +npm install apache-arrow # <-- combined es2015/CommonJS/ESModules/UMD + esnext/UMD +npm install @apache-arrow/ts # standalone TypeScript package +npm install @apache-arrow/es5-cjs # standalone es5/CommonJS package +npm install @apache-arrow/es5-esm # standalone es5/ESModules package +npm install @apache-arrow/es5-umd # standalone es5/UMD package +npm install @apache-arrow/es2015-cjs # standalone es2015/CommonJS package +npm install @apache-arrow/es2015-esm # standalone es2015/ESModules package +npm install @apache-arrow/es2015-umd # standalone es2015/UMD package +npm install @apache-arrow/esnext-cjs # standalone esNext/CommonJS package +npm install @apache-arrow/esnext-esm # standalone esNext/ESModules package +npm install @apache-arrow/esnext-umd # standalone esNext/UMD package +``` + +### Why we package like this + +The JS community is a diverse group with a varied list of target environments and tool chains. Publishing multiple packages accommodates projects of all stripes. + +If you think we missed a compilation target and it's a blocker for adoption, please open an issue. + +### Supported Browsers and Platforms + +The bundles we compile support moderns browser released in the last 5 years. This includes supported versions of +Firefox, Chrome, Edge, and Safari. We do not actively support Internet Explorer. +Apache Arrow also works on [maintained versions of Node](https://nodejs.org/en/about/releases/). + +# People + +Full list of broader Apache Arrow [committers](https://arrow.apache.org/committers/). + +* Brian Hulette, _committer_ +* Paul Taylor, _committer_ +* Dominik Moritz, _committer_ + +# Powered By Apache Arrow in JS + +Full list of broader Apache Arrow [projects & organizations](https://arrow.apache.org/powered_by/). + +## Open Source Projects + +* [Apache Arrow](https://arrow.apache.org) -- Parent project for Powering Columnar In-Memory Analytics, including affiliated open source projects +* [Perspective](https://github.com/jpmorganchase/perspective) -- Perspective is a streaming data visualization engine by J.P. Morgan for JavaScript for building real-time & user-configurable analytics entirely in the browser. +* [Falcon](https://github.com/uwdata/falcon) is a visualization tool for linked interactions across multiple aggregate visualizations of millions or billions of records. +* [Vega](https://github.com/vega) is an ecosystem of tools for interactive visualizations on the web. The Vega team implemented an [Arrow loader](https://github.com/vega/vega-loader-arrow). +* [Arquero](https://github.com/uwdata/arquero) is a library for query processing and transformation of array-backed data tables. +* [OmniSci](https://github.com/omnisci/mapd-connector) is a GPU database. Its JavaScript connector returns Arrow dataframes. + +# License + +[Apache 2.0](https://github.com/apache/arrow/blob/master/LICENSE) + +[1]: mailto:dev-subscribe@arrow.apache.org +[2]: https://github.com/apache/arrow/tree/master/format +[3]: https://issues.apache.org/jira/browse/ARROW +[4]: https://github.com/apache/arrow +[5]: https://beta.observablehq.com/@theneuralbit/introduction-to-apache-arrow +[6]: https://beta.observablehq.com/@lmeyerov/manipulating-flat-arrays-arrow-style +[7]: https://arrow.apache.org/docs/js/ +[8]: https://observablehq.com/@lmeyerov/rich-data-types-in-apache-arrow-js-efficient-data-tables-wit diff --git a/src/arrow/js/bin/arrow2csv.js b/src/arrow/js/bin/arrow2csv.js new file mode 100755 index 000000000..0e446fabe --- /dev/null +++ b/src/arrow/js/bin/arrow2csv.js @@ -0,0 +1,28 @@ +#! /usr/bin/env node + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const Path = require(`path`); +const here = Path.resolve(__dirname, '../'); +const tsnode = require.resolve(`ts-node/register`); +const arrow2csv = Path.join(here, `src/bin/arrow2csv.ts`); +const env = { ...process.env, TS_NODE_TRANSPILE_ONLY: `true` }; + +require('child_process').spawn(`node`, [ + `-r`, tsnode, arrow2csv, ...process.argv.slice(2) +], { cwd: here, env, stdio: `inherit` }); diff --git a/src/arrow/js/bin/file-to-stream.js b/src/arrow/js/bin/file-to-stream.js new file mode 100755 index 000000000..090cd0b0e --- /dev/null +++ b/src/arrow/js/bin/file-to-stream.js @@ -0,0 +1,40 @@ +#! /usr/bin/env node + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// @ts-check + +const fs = require('fs'); +const path = require('path'); +const eos = require('util').promisify(require('stream').finished); +const extension = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : ''; +const { RecordBatchReader, RecordBatchStreamWriter } = require(`../index${extension}`); + +(async () => { + + const readable = process.argv.length < 3 ? process.stdin : fs.createReadStream(path.resolve(process.argv[2])); + const writable = process.argv.length < 4 ? process.stdout : fs.createWriteStream(path.resolve(process.argv[3])); + + const fileToStream = readable + .pipe(RecordBatchReader.throughNode()) + .pipe(RecordBatchStreamWriter.throughNode()) + .pipe(writable); + + await eos(fileToStream); + +})().catch((e) => { console.error(e); process.exit(1); }); diff --git a/src/arrow/js/bin/integration.js b/src/arrow/js/bin/integration.js new file mode 100755 index 000000000..507514eba --- /dev/null +++ b/src/arrow/js/bin/integration.js @@ -0,0 +1,255 @@ +#! /usr/bin/env node + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// @ts-nocheck + +const fs = require('fs'); +const Path = require('path'); +const { promisify } = require('util'); +const glob = promisify(require('glob')); +const { zip } = require('ix/iterable/zip'); +const { parse: bignumJSONParse } = require('json-bignum'); +const argv = require(`command-line-args`)(cliOpts(), { partial: true }); +const { + Table, + RecordBatchReader, + util: { createElementComparator } +} = require('../targets/apache-arrow/'); + +const exists = async (p) => { + try { + return !!(await fs.promises.stat(p)); + } catch (e) { return false; } +} + +(async () => { + + if (!argv.mode) { return print_usage(); } + + let mode = argv.mode.toUpperCase(); + let jsonPaths = [...(argv.json || [])]; + let arrowPaths = [...(argv.arrow || [])]; + + if (mode === 'VALIDATE' && !jsonPaths.length) { + [jsonPaths, arrowPaths] = await loadLocalJSONAndArrowPathsForDebugging(jsonPaths, arrowPaths); + } + + if (!jsonPaths.length) { return print_usage(); } + + switch (mode) { + case 'VALIDATE': + for (let [jsonPath, arrowPath] of zip(jsonPaths, arrowPaths)) { + await validate(jsonPath, arrowPath); + } + break; + default: + return print_usage(); + } +})() +.then((x) => +x || 0, (e) => { + e && process.stderr.write(`${e?.stack || e}\n`); + return process.exitCode || 1; +}).then((code) => process.exit(code)); + +function cliOpts() { + return [ + { + type: String, + name: 'mode', + description: 'The integration test to run' + }, + { + type: String, + name: 'arrow', alias: 'a', + multiple: true, defaultValue: [], + description: 'The Arrow file[s] to read/write' + }, + { + type: String, + name: 'json', alias: 'j', + multiple: true, defaultValue: [], + description: 'The JSON file[s] to read/write' + } + ]; +} + +function print_usage() { + console.log(require('command-line-usage')([ + { + header: 'integration', + content: 'Script for running Arrow integration tests' + }, + { + header: 'Synopsis', + content: [ + '$ integration.js -j file.json -a file.arrow --mode validate' + ] + }, + { + header: 'Options', + optionList: [ + ...cliOpts(), + { + name: 'help', + description: 'Print this usage guide.' + } + ] + }, + ])); + return 1; +} + +async function validate(jsonPath, arrowPath) { + + const files = await Promise.all([ + fs.promises.readFile(arrowPath), + fs.promises.readFile(jsonPath, 'utf8'), + ]); + + const arrowData = files[0]; + const jsonData = bignumJSONParse(files[1]); + + validateReaderIntegration(jsonData, arrowData); + validateTableFromBuffersIntegration(jsonData, arrowData); + validateTableToBuffersIntegration('json', 'file')(jsonData, arrowData); + validateTableToBuffersIntegration('json', 'file')(jsonData, arrowData); + validateTableToBuffersIntegration('binary', 'file')(jsonData, arrowData); + validateTableToBuffersIntegration('binary', 'file')(jsonData, arrowData); +} + +function validateReaderIntegration(jsonData, arrowBuffer) { + const msg = `json and arrow record batches report the same values`; + try { + const jsonReader = RecordBatchReader.from(jsonData); + const binaryReader = RecordBatchReader.from(arrowBuffer); + for (const [jsonRecordBatch, binaryRecordBatch] of zip(jsonReader, binaryReader)) { + compareTableIsh(jsonRecordBatch, binaryRecordBatch); + } + } catch (e) { throw new Error(`${msg}: fail \n ${e?.stack || e}`); } + process.stdout.write(`${msg}: pass\n`); +} + +function validateTableFromBuffersIntegration(jsonData, arrowBuffer) { + const msg = `json and arrow tables report the same values`; + try { + const jsonTable = Table.from(jsonData); + const binaryTable = Table.from(arrowBuffer); + compareTableIsh(jsonTable, binaryTable); + } catch (e) { throw new Error(`${msg}: fail \n ${e?.stack || e}`); } + process.stdout.write(`${msg}: pass\n`); +} + +function validateTableToBuffersIntegration(srcFormat, arrowFormat) { + const refFormat = srcFormat === `json` ? `binary` : `json`; + return function testTableToBuffersIntegration(jsonData, arrowBuffer) { + const msg = `serialized ${srcFormat} ${arrowFormat} reports the same values as the ${refFormat} ${arrowFormat}`; + try { + const refTable = Table.from(refFormat === `json` ? jsonData : arrowBuffer); + const srcTable = Table.from(srcFormat === `json` ? jsonData : arrowBuffer); + const dstTable = Table.from(srcTable.serialize(`binary`, arrowFormat === `stream`)); + compareTableIsh(dstTable, refTable); + } catch (e) { throw new Error(`${msg}: fail \n ${e?.stack || e}`); } + process.stdout.write(`${msg}: pass\n`); + }; +} + +function compareTableIsh(actual, expected) { + if (actual.length !== expected.length) { + throw new Error(`length: ${actual.length} !== ${expected.length}`); + } + if (actual.numCols !== expected.numCols) { + throw new Error(`numCols: ${actual.numCols} !== ${expected.numCols}`); + } + (() => { + const getChildAtFn = expected instanceof Table ? 'getColumnAt' : 'getChildAt'; + for (let i = -1, n = actual.numCols; ++i < n;) { + const v1 = actual[getChildAtFn](i); + const v2 = expected[getChildAtFn](i); + compareVectors(v1, v2); + } + })(); +} + +function compareVectors(actual, expected) { + + if ((actual == null && expected != null) || (expected == null && actual != null)) { + throw new Error(`${actual == null ? `actual` : `expected`} is null, was expecting ${actual ?? expected} to be that also`); + } + + let props = ['type', 'length', 'nullCount']; + + (() => { + for (let i = -1, n = props.length; ++i < n;) { + const prop = props[i]; + if (`${actual[prop]}` !== `${expected[prop]}`) { + throw new Error(`${prop}: ${actual[prop]} !== ${expected[prop]}`); + } + } + })(); + + (() => { + for (let i = -1, n = actual.length; ++i < n;) { + let x1 = actual.get(i), x2 = expected.get(i); + if (!createElementComparator(x2)(x1)) { + throw new Error(`${i}: ${x1} !== ${x2}`); + } + } + })(); + + (() => { + let i = -1; + for (let [x1, x2] of zip(actual, expected)) { + ++i; + if (!createElementComparator(x2)(x1)) { + throw new Error(`${i}: ${x1} !== ${x2}`); + } + } + })(); +} + +async function loadLocalJSONAndArrowPathsForDebugging(jsonPaths, arrowPaths) { + + const sourceJSONPaths = await glob(Path.resolve(__dirname, `../test/data/json/`, `*.json`)); + + if (!arrowPaths.length) { + await loadJSONAndArrowPaths(sourceJSONPaths, jsonPaths, arrowPaths, 'cpp', 'file'); + await loadJSONAndArrowPaths(sourceJSONPaths, jsonPaths, arrowPaths, 'java', 'file'); + await loadJSONAndArrowPaths(sourceJSONPaths, jsonPaths, arrowPaths, 'cpp', 'stream'); + await loadJSONAndArrowPaths(sourceJSONPaths, jsonPaths, arrowPaths, 'java', 'stream'); + } + + for (let [jsonPath, arrowPath] of zip(jsonPaths, arrowPaths)) { + console.log(`jsonPath: ${jsonPath}`); + console.log(`arrowPath: ${arrowPath}`); + } + + return [jsonPaths, arrowPaths]; + + async function loadJSONAndArrowPaths(sourceJSONPaths, jsonPaths, arrowPaths, source, format) { + for (const jsonPath of sourceJSONPaths) { + const { name } = Path.parse(jsonPath); + const arrowPath = Path.resolve(__dirname, `../test/data/${source}/${format}/${name}.arrow`); + if (await exists(arrowPath)) { + jsonPaths.push(jsonPath); + arrowPaths.push(arrowPath); + } + } + return [jsonPaths, arrowPaths]; + } +} diff --git a/src/arrow/js/bin/json-to-arrow.js b/src/arrow/js/bin/json-to-arrow.js new file mode 100755 index 000000000..8f3fbd3fc --- /dev/null +++ b/src/arrow/js/bin/json-to-arrow.js @@ -0,0 +1,108 @@ +#! /usr/bin/env node + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// @ts-check + +const fs = require('fs'); +const Path = require('path'); +const { parse } = require('json-bignum'); +const eos = require('util').promisify(require('stream').finished); +const extension = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : ''; +const argv = require(`command-line-args`)(cliOpts(), { partial: true }); +const { RecordBatchReader, RecordBatchFileWriter, RecordBatchStreamWriter } = require(`../index${extension}`); + +const jsonPaths = [...(argv.json || [])]; +const arrowPaths = [...(argv.arrow || [])]; + +(async () => { + + if (!jsonPaths.length || !arrowPaths.length || (jsonPaths.length !== arrowPaths.length)) { + return print_usage(); + } + + await Promise.all(jsonPaths.map(async (path, i) => { + + const RecordBatchWriter = argv.format !== 'stream' + ? RecordBatchFileWriter + : RecordBatchStreamWriter; + + const reader = RecordBatchReader.from(parse( + await fs.promises.readFile(Path.resolve(path), 'utf8'))); + + const jsonToArrow = reader + .pipe(RecordBatchWriter.throughNode()) + .pipe(fs.createWriteStream(arrowPaths[i])); + + await eos(jsonToArrow); + + })); +})() +.then((x) => +x || 0, (e) => { + e && process.stderr.write(`${e}`); + return process.exitCode || 1; +}).then((code = 0) => process.exit(code)); + +function cliOpts() { + return [ + { + type: String, + name: 'format', alias: 'f', + multiple: false, defaultValue: 'file', + description: 'The Arrow format to write, either "file" or "stream"' + }, + { + type: String, + name: 'arrow', alias: 'a', + multiple: true, defaultValue: [], + description: 'The Arrow file[s] to write' + }, + { + type: String, + name: 'json', alias: 'j', + multiple: true, defaultValue: [], + description: 'The JSON file[s] to read' + } + ]; +} + +function print_usage() { + console.log(require('command-line-usage')([ + { + header: 'json-to-arrow', + content: 'Script for converting a JSON Arrow file to a binary Arrow file' + }, + { + header: 'Synopsis', + content: [ + '$ json-to-arrow.js -j in.json -a out.arrow -f stream' + ] + }, + { + header: 'Options', + optionList: [ + ...cliOpts(), + { + name: 'help', + description: 'Print this usage guide.' + } + ] + }, + ])); + return 1; +} diff --git a/src/arrow/js/bin/print-buffer-alignment.js b/src/arrow/js/bin/print-buffer-alignment.js new file mode 100755 index 000000000..4c3260397 --- /dev/null +++ b/src/arrow/js/bin/print-buffer-alignment.js @@ -0,0 +1,81 @@ +#! /usr/bin/env node + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// @ts-check + +const fs = require('fs'); +const path = require('path'); +const extension = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : ''; +const { RecordBatch, AsyncMessageReader } = require(`../index${extension}`); +const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`); + +(async () => { + + const readable = process.argv.length < 3 ? process.stdin : fs.createReadStream(path.resolve(process.argv[2])); + const reader = new AsyncMessageReader(readable); + + let schema, recordBatchIndex = 0, dictionaryBatchIndex = 0; + + for await (let message of reader) { + + let bufferRegions = []; + + if (message.isSchema()) { + schema = message.header(); + continue; + } else if (message.isRecordBatch()) { + const header = message.header(); + bufferRegions = header.buffers; + const body = await reader.readMessageBody(message.bodyLength); + const recordBatch = loadRecordBatch(schema, header, body); + console.log(`record batch ${++recordBatchIndex}: ${JSON.stringify({ + offset: body.byteOffset, + length: body.byteLength, + numRows: recordBatch.length, + })}`); + } else if (message.isDictionaryBatch()) { + const header = message.header(); + bufferRegions = header.data.buffers; + const type = schema.dictionaries.get(header.id); + const body = await reader.readMessageBody(message.bodyLength); + const recordBatch = loadDictionaryBatch(header.data, body, type); + console.log(`dictionary batch ${++dictionaryBatchIndex}: ${JSON.stringify({ + offset: body.byteOffset, + length: body.byteLength, + numRows: recordBatch.length, + dictionaryId: header.id, + })}`); + } + + bufferRegions.forEach(({ offset, length }, i) => { + console.log(`\tbuffer ${i + 1}: { offset: ${offset}, length: ${length} }`); + }); + } + + await reader.return(); + +})().catch((e) => { console.error(e); process.exit(1); }); + +function loadRecordBatch(schema, header, body) { + return new RecordBatch(schema, header.length, new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany(schema.fields)); +} + +function loadDictionaryBatch(header, body, dictionaryType) { + return RecordBatch.new(new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany([dictionaryType])); +} diff --git a/src/arrow/js/bin/stream-to-file.js b/src/arrow/js/bin/stream-to-file.js new file mode 100755 index 000000000..015a5eace --- /dev/null +++ b/src/arrow/js/bin/stream-to-file.js @@ -0,0 +1,40 @@ +#! /usr/bin/env node + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// @ts-check + +const fs = require('fs'); +const path = require('path'); +const eos = require('util').promisify(require('stream').finished); +const extension = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : ''; +const { RecordBatchReader, RecordBatchFileWriter } = require(`../index${extension}`); + +(async () => { + + const readable = process.argv.length < 3 ? process.stdin : fs.createReadStream(path.resolve(process.argv[2])); + const writable = process.argv.length < 4 ? process.stdout : fs.createWriteStream(path.resolve(process.argv[3])); + + const streamToFile = readable + .pipe(RecordBatchReader.throughNode()) + .pipe(RecordBatchFileWriter.throughNode()) + .pipe(writable); + + await eos(streamToFile); + +})().catch((e) => { console.error(e); process.exit(1); }); diff --git a/src/arrow/js/examples/read_file.html b/src/arrow/js/examples/read_file.html new file mode 100644 index 000000000..1013fbe79 --- /dev/null +++ b/src/arrow/js/examples/read_file.html @@ -0,0 +1,91 @@ +<!DOCTYPE html> + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<html> + <head> + <title>Arrow.js browser test</title> + <meta charset="utf-8"> + <style> +table { + border-collapse: collapse; +} +table, th, td { + border: 1px solid black; + white-space: nowrap; +} + </style> + <script type="text/javascript"> +var reader = new FileReader(); +function addCell (tr, type, value) { + var td = document.createElement(type) + td.textContent = value; + tr.appendChild(td); +} +reader.onload = function (evt) { + + var arrowTable = Arrow.Table.from([new Uint8Array(evt.target.result)]); + var thead = document.getElementById("thead"); + var tbody = document.getElementById("tbody"); + + while (thead.hasChildNodes()) { + thead.removeChild(thead.lastChild); + } + + while (tbody.hasChildNodes()) { + tbody.removeChild(tbody.lastChild); + } + + var header_row = document.createElement("tr"); + for (let field of arrowTable.schema.fields) { + addCell(header_row, "th", `${field}`); + } + + thead.appendChild(header_row); + + for (let row of arrowTable) { + var tr = document.createElement("tr"); + for (let cell of row) { + addCell(tr, "td", + cell == null ? 'null' + : !Array.isArray(cell) ? cell + : '[' + cell.map((value) => value == null ? 'null' : value).join(', ') + ']' + ); + } + tbody.appendChild(tr); + } +} + +function handleFiles(files) { + reader.readAsArrayBuffer(files[0]); +} + </script> + </head> + <body> + <input id="arrow-in" type="file" onchange="handleFiles(this.files)" /> + <table> + <thead id="thead"> + </thead> + <tbody id="tbody"> + </tbody> + </table> + <script type="text/javascript" src="../targets/es2015/umd/Arrow.js"></script> + </body> +</html> diff --git a/src/arrow/js/gulp/argv.js b/src/arrow/js/gulp/argv.js new file mode 100644 index 000000000..0acdad7d5 --- /dev/null +++ b/src/arrow/js/gulp/argv.js @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const argv = require(`command-line-args`)([ + { name: `all`, type: Boolean }, + { name: 'verbose', alias: `v`, type: Boolean }, + { name: `target`, type: String, defaultValue: `` }, + { name: `module`, type: String, defaultValue: `` }, + { name: `coverage`, type: Boolean, defaultValue: false }, + { name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] }, + { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] }, +], { partial: true }); + +const { targets, modules } = argv; + +if (argv.target === `src`) { + argv.target && !targets.length && targets.push(argv.target); +} else { + argv.target && !targets.length && targets.push(argv.target); + argv.module && !modules.length && modules.push(argv.module); + (argv.all || !targets.length) && targets.push(`all`); + (argv.all || !modules.length) && modules.push(`all`); +} + +module.exports = { argv, targets, modules }; diff --git a/src/arrow/js/gulp/arrow-task.js b/src/arrow/js/gulp/arrow-task.js new file mode 100644 index 000000000..fc85dd72e --- /dev/null +++ b/src/arrow/js/gulp/arrow-task.js @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { + targetDir, observableFromStreams +} = require('./util'); + +const del = require('del'); +const gulp = require('gulp'); +const mkdirp = require('mkdirp'); +const gulpRename = require(`gulp-rename`); +const { memoizeTask } = require('./memoize-task'); +const { + ReplaySubject, + forkJoin: ObservableForkJoin, +} = require('rxjs'); +const { + share +} = require('rxjs/operators'); +const pipeline = require('util').promisify(require('stream').pipeline); + +const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target) { + const out = targetDir(target); + const dtsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.ts`; + const cjsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.js`; + const esmGlob = `${targetDir(`es2015`, `esm`)}/**/*.js`; + const es2015UmdGlob = `${targetDir(`es2015`, `umd`)}/*.js`; + const esnextUmdGlob = `${targetDir(`esnext`, `umd`)}/*.js`; + const cjsSourceMapsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.map`; + const esmSourceMapsGlob = `${targetDir(`es2015`, `esm`)}/**/*.map`; + const es2015UmdSourceMapsGlob = `${targetDir(`es2015`, `umd`)}/*.map`; + const esnextUmdSourceMapsGlob = `${targetDir(`esnext`, `umd`)}/*.map`; + return ObservableForkJoin([ + observableFromStreams(gulp.src(dtsGlob), gulp.dest(out)), // copy d.ts files + observableFromStreams(gulp.src(cjsGlob), gulp.dest(out)), // copy es2015 cjs files + observableFromStreams(gulp.src(cjsSourceMapsGlob), gulp.dest(out)), // copy es2015 cjs sourcemaps + observableFromStreams(gulp.src(esmSourceMapsGlob), gulp.dest(out)), // copy es2015 esm sourcemaps + observableFromStreams(gulp.src(es2015UmdSourceMapsGlob), gulp.dest(out)), // copy es2015 umd sourcemap files, but don't rename + observableFromStreams(gulp.src(esnextUmdSourceMapsGlob), gulp.dest(out)), // copy esnext umd sourcemap files, but don't rename + observableFromStreams(gulp.src(esmGlob), gulpRename((p) => { p.extname = '.mjs'; }), gulp.dest(out)), // copy es2015 esm files and rename to `.mjs` + observableFromStreams(gulp.src(es2015UmdGlob), gulpRename((p) => { p.basename += `.es2015.min`; }), gulp.dest(out)), // copy es2015 umd files and add `.es2015.min` + observableFromStreams(gulp.src(esnextUmdGlob), gulpRename((p) => { p.basename += `.esnext.min`; }), gulp.dest(out)), // copy esnext umd files and add `.esnext.min` + ]).pipe(share({ connector: () => new ReplaySubject(), resetOnError: false, resetOnComplete: false, resetOnRefCountZero: false })); +}))({}); + +const arrowTSTask = ((cache) => memoizeTask(cache, async function copyTS(target, format) { + const out = targetDir(target, format); + await mkdirp(out); + await pipeline(gulp.src(`src/**/*`), gulp.dest(out)); + await del(`${out}/**/*.js`); +}))({}); + + +module.exports = arrowTask; +module.exports.arrowTask = arrowTask; +module.exports.arrowTSTask = arrowTSTask; diff --git a/src/arrow/js/gulp/clean-task.js b/src/arrow/js/gulp/clean-task.js new file mode 100644 index 000000000..0034f9a09 --- /dev/null +++ b/src/arrow/js/gulp/clean-task.js @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const del = require('del'); +const { targetDir } = require('./util'); +const memoizeTask = require('./memoize-task'); +const { catchError } = require('rxjs/operators'); +const { + from: ObservableFrom, + EMPTY: ObservableEmpty, +} = require('rxjs'); + +const cleanTask = ((cache) => memoizeTask(cache, function clean(target, format) { + const dir = targetDir(target, format); + return ObservableFrom(del(dir)) + .pipe(catchError((e) => ObservableEmpty())); +}))({}); + +module.exports = cleanTask; +module.exports.cleanTask = cleanTask;
\ No newline at end of file diff --git a/src/arrow/js/gulp/closure-task.js b/src/arrow/js/gulp/closure-task.js new file mode 100644 index 000000000..6e5a61d82 --- /dev/null +++ b/src/arrow/js/gulp/closure-task.js @@ -0,0 +1,213 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { + targetDir, + mainExport, + esmRequire, + gCCLanguageNames, + publicModulePaths, + observableFromStreams, + shouldRunInChildProcess, + spawnGulpCommandInChildProcess, +} = require('./util'); + +const fs = require('fs'); +const gulp = require('gulp'); +const path = require('path'); +const mkdirp = require('mkdirp'); +const sourcemaps = require('gulp-sourcemaps'); +const { memoizeTask } = require('./memoize-task'); +const { compileBinFiles } = require('./typescript-task'); +const closureCompiler = require('google-closure-compiler').gulp(); + +const closureTask = ((cache) => memoizeTask(cache, async function closure(target, format) { + + if (shouldRunInChildProcess(target, format)) { + return spawnGulpCommandInChildProcess('compile', target, format); + } + + const src = targetDir(target, `cls`); + const srcAbsolute = path.resolve(src); + const out = targetDir(target, format); + const externs = path.join(`${out}/${mainExport}.externs.js`); + const entry_point = path.join(`${src}/${mainExport}.dom.cls.js`); + + const exportedImports = publicModulePaths(srcAbsolute).reduce((entries, publicModulePath) => [ + ...entries, { + publicModulePath, + exports_: getPublicExportedNames(esmRequire(publicModulePath)) + } + ], []); + + await mkdirp(out); + + await Promise.all([ + fs.promises.writeFile(externs, generateExternsFile(exportedImports)), + fs.promises.writeFile(entry_point, generateUMDExportAssignment(srcAbsolute, exportedImports)) + ]); + + return await Promise.all([ + runClosureCompileAsObservable().toPromise(), + compileBinFiles(target, format).toPromise() + ]); + + function runClosureCompileAsObservable() { + return observableFromStreams( + gulp.src([ + /* external libs first */ + `node_modules/flatbuffers/package.json`, + `node_modules/flatbuffers/js/flatbuffers.mjs`, + `${src}/**/*.js` /* <-- then source globs */ + ], { base: `./` }), + sourcemaps.init(), + closureCompiler(createClosureArgs(entry_point, externs, target), { + platform: ['native', 'java', 'javascript'] + }), + // rename the sourcemaps from *.js.map files to *.min.js.map + sourcemaps.write(`.`, { mapFile: (mapPath) => mapPath.replace(`.js.map`, `.${target}.min.js.map`) }), + gulp.dest(out) + ); + } +}))({}); + +module.exports = closureTask; +module.exports.closureTask = closureTask; + +const createClosureArgs = (entry_point, externs, target) => ({ + externs, + entry_point, + third_party: true, + warning_level: `QUIET`, + dependency_mode: `PRUNE`, + rewrite_polyfills: false, + module_resolution: `NODE`, + // formatting: `PRETTY_PRINT`, + // debug: true, + compilation_level: `ADVANCED`, + package_json_entry_names: `module,jsnext:main,main`, + assume_function_wrapper: true, + js_output_file: `${mainExport}.js`, + language_in: gCCLanguageNames[`esnext`], + language_out: gCCLanguageNames[target], + output_wrapper:`${apacheHeader()} +(function (global, factory) { + typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) : + typeof define === 'function' && define.amd ? define(['exports'], factory) : + (factory(global.Arrow = global.Arrow || {})); +}(this, (function (exports) {%output%}.bind(this))));` +}); + +function generateUMDExportAssignment(src, exportedImports) { + return [ + ...exportedImports.map(({ publicModulePath }, i) => { + const p = publicModulePath.slice(src.length + 1); + return (`import * as exports${i} from './${p}';`); + }).filter(Boolean), + 'Object.assign(arguments[0], exports0);' + ].join('\n'); +} + +function generateExternsFile(exportedImports) { + return [ + externsHeader(), + ...exportedImports.reduce((externBodies, { exports_ }) => [ + ...externBodies, ...exports_.map(externBody) + ], []).filter(Boolean) + ].join('\n'); +} + +function externBody({ exportName, staticNames, instanceNames }) { + return [ + `var ${exportName} = function() {};`, + staticNames.map((staticName) => (isNaN(+staticName) + ? `/** @type {?} */\n${exportName}.${staticName} = function() {};` + : `/** @type {?} */\n${exportName}[${staticName}] = function() {};` + )).join('\n'), + instanceNames.map((instanceName) => (isNaN(+instanceName) + ? `/** @type {?} */\n${exportName}.prototype.${instanceName};` + : `/** @type {?} */\n${exportName}.prototype[${instanceName}];` + )).join('\n') + ].filter(Boolean).join('\n'); +} + +function externsHeader() { + return (`${apacheHeader()} +// @ts-nocheck +/* eslint-disable */ +/** + * @fileoverview Closure Compiler externs for Arrow + * @externs + * @suppress {duplicate,checkTypes} + */ +/** @type {symbol} */ +Symbol.iterator; +/** @type {symbol} */ +Symbol.toPrimitive; +/** @type {symbol} */ +Symbol.asyncIterator; +`); +} + +function getPublicExportedNames(entryModule) { + const fn = function() {}; + const isStaticOrProtoName = (x) => ( + !(x in fn) && + (x !== `default`) && + (x !== `undefined`) && + (x !== `__esModule`) && + (x !== `constructor`) && + !(x.startsWith('_')) + ); + return Object + .getOwnPropertyNames(entryModule) + .filter((name) => name !== 'default') + .filter((name) => ( + typeof entryModule[name] === `object` || + typeof entryModule[name] === `function` + )) + .map((name) => [name, entryModule[name]]) + .reduce((reserved, [name, value]) => { + + const staticNames = value && + typeof value === 'object' ? Object.getOwnPropertyNames(value).filter(isStaticOrProtoName) : + typeof value === 'function' ? Object.getOwnPropertyNames(value).filter(isStaticOrProtoName) : []; + + const instanceNames = (typeof value === `function` && Object.getOwnPropertyNames(value.prototype || {}) || []).filter(isStaticOrProtoName); + + return [...reserved, { exportName: name, staticNames, instanceNames }]; + }, []); +} + +function apacheHeader() { + return `// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License.`; +} diff --git a/src/arrow/js/gulp/compile-task.js b/src/arrow/js/gulp/compile-task.js new file mode 100644 index 000000000..07109ef73 --- /dev/null +++ b/src/arrow/js/gulp/compile-task.js @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { Observable } = require('rxjs'); +const { npmPkgName } = require('./util'); +const { memoizeTask } = require('./memoize-task'); + +const closureTask = require('./closure-task'); +const typescriptTask = require('./typescript-task'); +const { arrowTask, arrowTSTask } = require('./arrow-task'); + +const compileTask = ((cache) => memoizeTask(cache, function compile(target, format, ...args) { + return target === `src` ? Observable.empty() + : target === npmPkgName ? arrowTask(target, format, ...args)() + : target === `ts` ? arrowTSTask(target, format, ...args)() + : format === `umd` ? closureTask(target, format, ...args)() + : typescriptTask(target, format, ...args)(); +}))({}); + +module.exports = compileTask; +module.exports.compileTask = compileTask; diff --git a/src/arrow/js/gulp/memoize-task.js b/src/arrow/js/gulp/memoize-task.js new file mode 100644 index 000000000..408ee3b88 --- /dev/null +++ b/src/arrow/js/gulp/memoize-task.js @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { taskName } = require('./util'); + +const createTask = ((taskFn) => ((target, format, ...args) => { + // Give the memoized fn a displayName so gulp's output is easier to follow. + const fn = () => taskFn(target, format, ...args); + fn.displayName = `${taskFn.name || ``}:${taskName(target, format, ...args)}:task`; + return fn; +})); + +const memoizeTask = ((cache, taskFn) => ((target, format, ...args) => { + // Give the memoized fn a displayName so gulp's output is easier to follow. + const fn = () => ( + cache[taskName(target, format)] || ( + cache[taskName(target, format)] = taskFn(target, format, ...args))); + fn.displayName = `${taskFn.name || ``}:${taskName(target, format, ...args)}:task`; + return fn; +})); + +module.exports = memoizeTask; +module.exports.createTask = createTask; +module.exports.memoizeTask = memoizeTask; diff --git a/src/arrow/js/gulp/package-task.js b/src/arrow/js/gulp/package-task.js new file mode 100644 index 000000000..321e65a30 --- /dev/null +++ b/src/arrow/js/gulp/package-task.js @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { + metadataFiles, packageJSONFields, + mainExport, npmPkgName, npmOrgName, + targetDir, packageName, observableFromStreams +} = require('./util'); + +const gulp = require('gulp'); +const { memoizeTask } = require('./memoize-task'); +const { + ReplaySubject, + EMPTY: ObservableEmpty, + forkJoin: ObservableForkJoin, +} = require('rxjs'); +const { + share +} = require('rxjs/operators'); +const gulpJsonTransform = require('gulp-json-transform'); + +const packageTask = ((cache) => memoizeTask(cache, function bundle(target, format) { + if (target === `src`) return ObservableEmpty(); + const out = targetDir(target, format); + const jsonTransform = gulpJsonTransform(target === npmPkgName ? createMainPackageJson(target, format) : + target === `ts` ? createTypeScriptPackageJson(target, format) + : createScopedPackageJSON(target, format), + 2); + return ObservableForkJoin([ + observableFromStreams(gulp.src(metadataFiles), gulp.dest(out)), // copy metadata files + observableFromStreams(gulp.src(`package.json`), jsonTransform, gulp.dest(out)) // write packageJSONs + ]).pipe(share({ connector: () => new ReplaySubject(), resetOnError: false, resetOnComplete: false, resetOnRefCountZero: false })); +}))({}); + +module.exports = packageTask; +module.exports.packageTask = packageTask; + +// FIXME: set this to false when we have no side effects +const sideEffects = true; + +const createMainPackageJson = (target, format) => (orig) => ({ + ...createTypeScriptPackageJson(target, format)(orig), + bin: orig.bin, + name: npmPkgName, + type: 'commonjs', + main: `${mainExport}.node.js`, + module: `${mainExport}.node.mjs`, + browser: { + [`./${mainExport}.node.js`]: `./${mainExport}.dom.js`, + [`./${mainExport}.node.mjs`]: `./${mainExport}.dom.mjs` + }, + exports: { + import: `./${mainExport}.node.mjs`, + require: `./${mainExport}.node.js`, + }, + types: `${mainExport}.node.d.ts`, + unpkg: `${mainExport}.es2015.min.js`, + jsdelivr: `${mainExport}.es2015.min.js`, + sideEffects: sideEffects, + esm: { mode: `all`, sourceMap: true } +}); + +const createTypeScriptPackageJson = (target, format) => (orig) => ({ + ...createScopedPackageJSON(target, format)(orig), + bin: undefined, + main: `${mainExport}.node.ts`, + module: `${mainExport}.node.ts`, + types: `${mainExport}.node.ts`, + browser: `${mainExport}.dom.ts`, + type: "module", + sideEffects: sideEffects, + esm: { mode: `auto`, sourceMap: true }, + dependencies: { + '@types/flatbuffers': '*', + '@types/node': '*', + ...orig.dependencies + } +}); + +const createScopedPackageJSON = (target, format) => (({ name, ...orig }) => + packageJSONFields.reduce( + (xs, key) => ({ ...xs, [key]: xs[key] || orig[key] }), + { + // un-set version, since it's automatically applied during the release process + version: undefined, + // set the scoped package name (e.g. "@apache-arrow/esnext-esm") + name: `${npmOrgName}/${packageName(target, format)}`, + // set "unpkg"/"jsdeliver" if building scoped UMD target + unpkg: format === 'umd' ? `${mainExport}.js` : undefined, + jsdelivr: format === 'umd' ? `${mainExport}.js` : undefined, + // set "browser" if building scoped UMD target, otherwise "Arrow.dom" + browser: format === 'umd' ? `${mainExport}.js` : `${mainExport}.dom.js`, + // set "main" to "Arrow" if building scoped UMD target, otherwise "Arrow.node" + main: format === 'umd' ? `${mainExport}.js` : `${mainExport}.node.js`, + // set "type" to `module` or `commonjs` (https://nodejs.org/api/packages.html#packages_type) + type: format === 'esm' ? `module` : `commonjs`, + // set "module" if building scoped ESM target + module: format === 'esm' ? `${mainExport}.node.js` : undefined, + // set "sideEffects" to false as a hint to Webpack that it's safe to tree-shake the ESM target + sideEffects: format === 'esm' ? sideEffects : undefined, + // include "esm" settings for https://www.npmjs.com/package/esm if building scoped ESM target + esm: format === `esm` ? { mode: `auto`, sourceMap: true } : undefined, + // set "types" (for TypeScript/VSCode) + types: format === 'umd' ? undefined : `${mainExport}.node.d.ts`, + } + ) +); diff --git a/src/arrow/js/gulp/test-task.js b/src/arrow/js/gulp/test-task.js new file mode 100644 index 000000000..2012f7429 --- /dev/null +++ b/src/arrow/js/gulp/test-task.js @@ -0,0 +1,186 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const del = require('del'); +const path = require('path'); +const mkdirp = require('mkdirp'); +const cpy = require('cpy'); +const { argv } = require('./argv'); +const { promisify } = require('util'); +const glob = promisify(require('glob')); +const child_process = require(`child_process`); +const { memoizeTask } = require('./memoize-task'); +const readFile = promisify(require('fs').readFile); +const asyncDone = promisify(require('async-done')); +const exec = promisify(require('child_process').exec); +const parseXML = promisify(require('xml2js').parseString); +const { targetAndModuleCombinations, npmPkgName } = require('./util'); + +const jestArgv = [`--reporters=jest-silent-reporter`]; + +if (argv.verbose) { + jestArgv.push(`--verbose`); +} + +if (targetAndModuleCombinations.length > 1) { + jestArgv.push(`--detectOpenHandles`); +} + +const jest = path.join(path.parse(require.resolve(`jest`)).dir, `../bin/jest.js`); +const testOptions = { + stdio: [`ignore`, `inherit`, `inherit`], + env: { + ...process.env, + // hide fs.promises/stream[Symbol.asyncIterator] warnings + NODE_NO_WARNINGS: `1`, + }, +}; + +const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function test(target, format) { + const opts = { ...testOptions }; + const args = [...execArgv]; + if (format === 'esm' || target === 'ts' || target === 'src' || target === npmPkgName) { + args.unshift(`--experimental-vm-modules`); + } + if (argv.coverage) { + args.push(`-c`, `jestconfigs/jest.coverage.config.js`); + } else { + const cfgname = [target, format].filter(Boolean).join('.'); + args.push(`-c`, `jestconfigs/jest.${cfgname}.config.js`, `test/unit/`); + } + opts.env = { + ...opts.env, + TEST_TARGET: target, + TEST_MODULE: format, + TEST_DOM_STREAMS: (target ==='src' || format === 'umd').toString(), + TEST_NODE_STREAMS: (target ==='src' || format !== 'umd').toString(), + TEST_TS_SOURCE: !!argv.coverage || (target === 'src') || (opts.env.TEST_TS_SOURCE === 'true') + }; + return asyncDone(() => child_process.spawn(`node`, args, opts)); +}))({}, [jest, ...jestArgv], testOptions); + +module.exports = testTask; +module.exports.testTask = testTask; +module.exports.cleanTestData = cleanTestData; +module.exports.createTestData = createTestData; + +// Pull C++ and Java paths from environment vars first, otherwise sane defaults +const ARROW_HOME = process.env.ARROW_HOME || path.resolve('../'); +const ARROW_JAVA_DIR = process.env.ARROW_JAVA_DIR || path.join(ARROW_HOME, 'java'); +const CPP_EXE_PATH = process.env.ARROW_CPP_EXE_PATH || path.join(ARROW_HOME, 'cpp/build/debug'); +const ARROW_INTEGRATION_DIR = process.env.ARROW_INTEGRATION_DIR || path.join(ARROW_HOME, 'integration'); +const CPP_JSON_TO_ARROW = path.join(CPP_EXE_PATH, 'arrow-json-integration-test'); +const CPP_FILE_TO_STREAM = path.join(CPP_EXE_PATH, 'arrow-file-to-stream'); + +const testFilesDir = path.join(ARROW_HOME, 'js/test/data'); +const snapshotsDir = path.join(ARROW_HOME, 'js/test/__snapshots__'); +const cppFilesDir = path.join(testFilesDir, 'cpp'); +const javaFilesDir = path.join(testFilesDir, 'java'); +const jsonFilesDir = path.join(testFilesDir, 'json'); + +async function cleanTestData() { + return await del([ + `${cppFilesDir}/**`, + `${javaFilesDir}/**`, + `${jsonFilesDir}/**`, + `${snapshotsDir}/**` + ]); +} + +async function createTestJSON() { + await mkdirp(jsonFilesDir); + await cpy(`cp ${ARROW_INTEGRATION_DIR}/data/*.json`, jsonFilesDir); + await exec(`python3 ${ARROW_INTEGRATION_DIR}/integration_test.py --write_generated_json ${jsonFilesDir}`); +} + +async function createTestData() { + + let JAVA_TOOLS_JAR = process.env.ARROW_JAVA_INTEGRATION_JAR; + if (!JAVA_TOOLS_JAR) { + const pom_version = await + readFile(path.join(ARROW_JAVA_DIR, 'pom.xml')) + .then((pom) => parseXML(pom.toString())) + .then((pomXML) => pomXML.project.version[0]); + JAVA_TOOLS_JAR = path.join(ARROW_JAVA_DIR, `/tools/target/arrow-tools-${pom_version}-jar-with-dependencies.jar`); + } + + await cleanTestData().then(createTestJSON); + await mkdirp(path.join(cppFilesDir, 'file')); + await mkdirp(path.join(javaFilesDir, 'file')); + await mkdirp(path.join(cppFilesDir, 'stream')); + await mkdirp(path.join(javaFilesDir, 'stream')); + + const errors = []; + const names = await glob(path.join(jsonFilesDir, '*.json')); + + for (let jsonPath of names) { + const name = path.parse(path.basename(jsonPath)).name; + const arrowCppFilePath = path.join(cppFilesDir, 'file', `${name}.arrow`); + const arrowJavaFilePath = path.join(javaFilesDir, 'file', `${name}.arrow`); + const arrowCppStreamPath = path.join(cppFilesDir, 'stream', `${name}.arrow`); + const arrowJavaStreamPath = path.join(javaFilesDir, 'stream', `${name}.arrow`); + try { + await generateCPPFile(path.resolve(jsonPath), arrowCppFilePath); + await generateCPPStream(arrowCppFilePath, arrowCppStreamPath); + } catch (e) { errors.push(`${e.stdout}\n${e.message}`); } + try { + await generateJavaFile(path.resolve(jsonPath), arrowJavaFilePath); + await generateJavaStream(arrowJavaFilePath, arrowJavaStreamPath); + } catch (e) { errors.push(`${e.stdout}\n${e.message}`); } + } + if (errors.length) { + console.error(errors.join(`\n`)); + process.exit(1); + } + + async function generateCPPFile(jsonPath, filePath) { + await del(filePath); + return await exec( + `${CPP_JSON_TO_ARROW} ${ + `--integration --mode=JSON_TO_ARROW`} ${ + `--json=${jsonPath} --arrow=${filePath}`}`, + { maxBuffer: Math.pow(2, 53) - 1 } + ); + } + + async function generateCPPStream(filePath, streamPath) { + await del(streamPath); + return await exec( + `${CPP_FILE_TO_STREAM} ${filePath} > ${streamPath}`, + { maxBuffer: Math.pow(2, 53) - 1 } + ); + } + + async function generateJavaFile(jsonPath, filePath) { + await del(filePath); + return await exec( + `java -cp ${JAVA_TOOLS_JAR} ${ + `org.apache.arrow.tools.Integration -c JSON_TO_ARROW`} ${ + `-j ${path.resolve(jsonPath)} -a ${filePath}`}`, + { maxBuffer: Math.pow(2, 53) - 1 } + ); + } + + async function generateJavaStream(filePath, streamPath) { + await del(streamPath); + return await exec( + `java -cp ${JAVA_TOOLS_JAR} ${ + `org.apache.arrow.tools.FileToStream`} ${filePath} ${streamPath}`, + { maxBuffer: Math.pow(2, 53) - 1 } + ); + } +} diff --git a/src/arrow/js/gulp/typescript-task.js b/src/arrow/js/gulp/typescript-task.js new file mode 100644 index 000000000..ed03b8453 --- /dev/null +++ b/src/arrow/js/gulp/typescript-task.js @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { + targetDir, + tsconfigName, + observableFromStreams, + shouldRunInChildProcess, + spawnGulpCommandInChildProcess, +} = require('./util'); + +const gulp = require('gulp'); +const path = require('path'); +const ts = require(`gulp-typescript`); +const sourcemaps = require('gulp-sourcemaps'); +const { memoizeTask } = require('./memoize-task'); +const { + ReplaySubject, + forkJoin: ObservableForkJoin, +} = require('rxjs'); +const { + mergeWith, + takeLast, + share +} = require('rxjs/operators'); + +const typescriptTask = ((cache) => memoizeTask(cache, function typescript(target, format) { + if (shouldRunInChildProcess(target, format)) { + return spawnGulpCommandInChildProcess('compile', target, format); + } + + const out = targetDir(target, format); + const tsconfigPath = path.join(`tsconfig`, `tsconfig.${tsconfigName(target, format)}.json`); + return compileTypescript(out, tsconfigPath) + .pipe(mergeWith(compileBinFiles(target, format))) + .pipe(takeLast(1)) + .pipe(share({ connector: () => new ReplaySubject(), resetOnError: false, resetOnComplete: false, resetOnRefCountZero: false })) +}))({}); + +function compileBinFiles(target, format) { + const out = targetDir(target, format); + const tsconfigPath = path.join(`tsconfig`, `tsconfig.${tsconfigName('bin', 'cjs')}.json`); + return compileTypescript(path.join(out, 'bin'), tsconfigPath, { target }); +} + +function compileTypescript(out, tsconfigPath, tsconfigOverrides) { + const tsProject = ts.createProject(tsconfigPath, { typescript: require(`typescript`), ...tsconfigOverrides}); + const { stream: { js, dts } } = observableFromStreams( + tsProject.src(), sourcemaps.init(), + tsProject(ts.reporter.defaultReporter()) + ); + const writeSources = observableFromStreams(tsProject.src(), gulp.dest(path.join(out, 'src'))); + const writeDTypes = observableFromStreams(dts, sourcemaps.write('./', { includeContent: false, sourceRoot: 'src' }), gulp.dest(out)); + const mapFile = tsProject.options.module === 5 ? esmMapFile : cjsMapFile; + const writeJS = observableFromStreams(js, sourcemaps.write('./', { mapFile, includeContent: false }), gulp.dest(out)); + return ObservableForkJoin([writeSources, writeDTypes, writeJS]); +} + +function cjsMapFile(mapFilePath) { return mapFilePath; } +function esmMapFile(mapFilePath) { return mapFilePath.replace('.js.map', '.mjs.map'); } + +module.exports = typescriptTask; +module.exports.typescriptTask = typescriptTask; +module.exports.compileBinFiles = compileBinFiles; diff --git a/src/arrow/js/gulp/util.js b/src/arrow/js/gulp/util.js new file mode 100644 index 000000000..d8cde29e8 --- /dev/null +++ b/src/arrow/js/gulp/util.js @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const fs = require('fs'); +const path = require(`path`); +const pump = require(`stream`).pipeline; +const child_process = require(`child_process`); +const { targets, modules } = require('./argv'); +const { + ReplaySubject, + empty: ObservableEmpty, + throwError: ObservableThrow, + fromEvent: ObservableFromEvent +} = require('rxjs'); +const { + share, + flatMap, + takeUntil, + defaultIfEmpty, + mergeWith, +} = require('rxjs/operators'); +const asyncDone = require('util').promisify(require('async-done')); + +const mainExport = `Arrow`; +const npmPkgName = `apache-arrow`; +const npmOrgName = `@${npmPkgName}`; + +const releasesRootDir = `targets`; +const knownTargets = [`es5`, `es2015`, `esnext`]; +const knownModules = [`cjs`, `esm`, `cls`, `umd`]; +const tasksToSkipPerTargetOrFormat = { + src: { clean: true, build: true }, + cls: { test: true, package: true } +}; +const packageJSONFields = [ + `version`, `license`, `description`, + `author`, `homepage`, `repository`, + `bugs`, `keywords`, `dependencies`, + `bin` +]; + +const metadataFiles = [`LICENSE.txt`, `NOTICE.txt`, `README.md`].map((filename) => { + let prefixes = [`./`, `../`]; + let p = prefixes.find((prefix) => { + try { + fs.statSync(path.resolve(path.join(prefix, filename))); + } catch (e) { return false; } + return true; + }); + if (!p) { + throw new Error(`Couldn't find ${filename} in ./ or ../`); + } + return path.join(p, filename); +}); + +// see: https://github.com/google/closure-compiler/blob/c1372b799d94582eaf4b507a4a22558ff26c403c/src/com/google/javascript/jscomp/CompilerOptions.java#L2988 +const gCCLanguageNames = { + es5: `ECMASCRIPT5`, + es2015: `ECMASCRIPT_2015`, + es2016: `ECMASCRIPT_2016`, + es2017: `ECMASCRIPT_2017`, + es2018: `ECMASCRIPT_2018`, + es2019: `ECMASCRIPT_2019`, + esnext: `ECMASCRIPT_NEXT` +}; + +function taskName(target, format) { + return !format ? target : `${target}:${format}`; +} + +function packageName(target, format) { + return !format ? target : `${target}-${format}`; +} + +function tsconfigName(target, format) { + return !format ? target : `${target}.${format}`; +} + +function targetDir(target, format) { + return path.join(releasesRootDir, ...(!format ? [target] : [target, format])); +} + +function shouldRunInChildProcess(target, format) { + // If we're building more than one module/target, then yes run this task in a child process + if (targets.length > 1 || modules.length > 1) { return true; } + // If the target we're building *isn't* the target the gulp command was configured to run, then yes run that in a child process + if (targets[0] !== target || modules[0] !== format) { return true; } + // Otherwise no need -- either gulp was run for just one target, or we've been spawned as the child of a multi-target parent gulp + return false; +} + +const gulp = path.join(path.parse(require.resolve(`gulp`)).dir, `bin/gulp.js`); +function spawnGulpCommandInChildProcess(command, target, format) { + const args = [gulp, command, '-t', target, '-m', format, `--silent`]; + const opts = { + stdio: [`ignore`, `inherit`, `inherit`], + env: { ...process.env, NODE_NO_WARNINGS: `1` } + }; + return asyncDone(() => child_process.spawn(`node`, args, opts)) + .catch((e) => { throw `Error in "${command}:${taskName(target, format)}" task`; }); +} + +const logAndDie = (e) => { if (e) { process.exit(1) } }; +function observableFromStreams(...streams) { + if (streams.length <= 0) { return ObservableEmpty(); } + const pumped = streams.length <= 1 ? streams[0] : pump(...streams, logAndDie); + const fromEvent = ObservableFromEvent.bind(null, pumped); + const streamObs = fromEvent(`data`).pipe( + mergeWith(fromEvent(`error`).pipe(flatMap((e) => ObservableThrow(e)))), + takeUntil(fromEvent(`end`).pipe(mergeWith(fromEvent(`close`)))), + defaultIfEmpty(`empty stream`), + share({ connector: () => new ReplaySubject(), resetOnError: false, resetOnComplete: false, resetOnRefCountZero: false }) + ); + streamObs.stream = pumped; + streamObs.observable = streamObs; + return streamObs; +} + +function* combinations(_targets, _modules) { + const targets = known(knownTargets, _targets || [`all`]); + const modules = known(knownModules, _modules || [`all`]); + + if (_targets.includes(`src`)) { + yield [`src`, ``]; + return; + } + + if (_targets.includes(`all`) && _modules.includes(`all`)) { + yield [`ts`, ``]; + yield [`src`, ``]; + yield [npmPkgName, ``]; + } + + for (const format of modules) { + for (const target of targets) { + yield [target, format]; + } + } + + function known(known, values) { + return values.includes(`all`) ? known + : values.includes(`src`) ? [`src`] + : Object.keys( + values.reduce((map, arg) => (( + (known.includes(arg)) && + (map[arg.toLowerCase()] = true) + || true) && map + ), {}) + ).sort((a, b) => known.indexOf(a) - known.indexOf(b)); + } +} + +const publicModulePaths = (dir) => [ + `${dir}/${mainExport}.dom.js`, + `${dir}/util/int.js`, + `${dir}/compute/predicate.js`, +]; + +const esmRequire = require(`esm`)(module, { + mode: `auto`, + cjs: { + /* A boolean for storing ES modules in require.cache. */ + cache: true, + /* A boolean for respecting require.extensions in ESM. */ + extensions: true, + /* A boolean for __esModule interoperability. */ + interop: true, + /* A boolean for importing named exports of CJS modules. */ + namedExports: true, + /* A boolean for following CJS path rules in ESM. */ + paths: true, + /* A boolean for __dirname, __filename, and require in ESM. */ + vars: true, + } +}); + +module.exports = { + mainExport, npmPkgName, npmOrgName, metadataFiles, packageJSONFields, + + knownTargets, knownModules, tasksToSkipPerTargetOrFormat, gCCLanguageNames, + + taskName, packageName, tsconfigName, targetDir, combinations, observableFromStreams, + publicModulePaths, esmRequire, shouldRunInChildProcess, spawnGulpCommandInChildProcess, + + targetAndModuleCombinations: [...combinations(targets, modules)] +}; diff --git a/src/arrow/js/gulpfile.js b/src/arrow/js/gulpfile.js new file mode 100644 index 000000000..a257a2def --- /dev/null +++ b/src/arrow/js/gulpfile.js @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const del = require('del'); +const gulp = require('gulp'); +const { targets } = require('./gulp/argv'); +const { + from: ObservableFrom, + bindNodeCallback: ObservableBindNodeCallback +} = require('rxjs'); +const { flatMap } = require('rxjs/operators'); +const cleanTask = require('./gulp/clean-task'); +const compileTask = require('./gulp/compile-task'); +const packageTask = require('./gulp/package-task'); +const { testTask, createTestData, cleanTestData } = require('./gulp/test-task'); +const { + taskName, combinations, + targetDir, knownTargets, + npmPkgName, tasksToSkipPerTargetOrFormat, + targetAndModuleCombinations +} = require('./gulp/util'); + +for (const [target, format] of combinations([`all`], [`all`])) { + const task = taskName(target, format); + gulp.task(`clean:${task}`, cleanTask(target, format)); + gulp.task(`test:${task}`, testTask(target, format)); + gulp.task(`compile:${task}`, compileTask(target, format)); + gulp.task(`package:${task}`, packageTask(target, format)); + gulp.task(`build:${task}`, gulp.series( + `clean:${task}`, `compile:${task}`, `package:${task}` + )); +} + +// The UMD bundles build temporary es5/6/next targets via TS, +// then run the TS source through either closure-compiler or +// a minifier, so we special case that here. +knownTargets.forEach((target) => { + const umd = taskName(target, `umd`); + const cls = taskName(target, `cls`); + gulp.task(`build:${umd}`, gulp.series( + `build:${cls}`, + `clean:${umd}`, `compile:${umd}`, `package:${umd}`, + function remove_closure_tmp_files() { + return del(targetDir(target, `cls`)) + } + )); +}); + +// The main "apache-arrow" module builds the es2015/umd, es2015/cjs, +// es2015/esm, and esnext/umd targets, then copies and renames the +// compiled output into the apache-arrow folder +gulp.task(`build:${npmPkgName}`, + gulp.series( + gulp.parallel( + `build:${taskName(`es2015`, `umd`)}`, + `build:${taskName(`es2015`, `cjs`)}`, + `build:${taskName(`es2015`, `esm`)}`, + `build:${taskName(`esnext`, `umd`)}` + ), + `clean:${npmPkgName}`, + `compile:${npmPkgName}`, + `package:${npmPkgName}` + ) +); + +// And finally the global composite tasks +gulp.task(`clean:testdata`, cleanTestData); +gulp.task(`create:testdata`, createTestData); +gulp.task(`test`, gulpConcurrent(getTasks(`test`))); +gulp.task(`clean`, gulp.parallel(getTasks(`clean`))); +gulp.task(`build`, gulpConcurrent(getTasks(`build`))); +gulp.task(`compile`, gulpConcurrent(getTasks(`compile`))); +gulp.task(`package`, gulpConcurrent(getTasks(`package`))); +gulp.task(`default`, gulp.series(`clean`, `build`, `test`)); + +function gulpConcurrent(tasks, numCPUs = Math.max(1, require('os').cpus().length * 0.5) | 0) { + return () => ObservableFrom(tasks.map((task) => gulp.series(task))) + .pipe(flatMap((task) => ObservableBindNodeCallback(task)(), numCPUs || 1)); +} + +function getTasks(name) { + const tasks = []; + if (targets.includes(`ts`)) tasks.push(`${name}:ts`); + if (targets.includes(npmPkgName)) tasks.push(`${name}:${npmPkgName}`); + for (const [target, format] of targetAndModuleCombinations) { + if (tasksToSkipPerTargetOrFormat[target] && tasksToSkipPerTargetOrFormat[target][name]) continue; + if (tasksToSkipPerTargetOrFormat[format] && tasksToSkipPerTargetOrFormat[format][name]) continue; + tasks.push(`${name}:${taskName(target, format)}`); + } + return tasks.length && tasks || [(done) => done()]; +} diff --git a/src/arrow/js/index.js b/src/arrow/js/index.js new file mode 100644 index 000000000..e42cb32d1 --- /dev/null +++ b/src/arrow/js/index.js @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = require('./targets/apache-arrow');
\ No newline at end of file diff --git a/src/arrow/js/index.mjs b/src/arrow/js/index.mjs new file mode 100644 index 000000000..304353712 --- /dev/null +++ b/src/arrow/js/index.mjs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +export * from './targets/apache-arrow';
\ No newline at end of file diff --git a/src/arrow/js/index.ts b/src/arrow/js/index.ts new file mode 100644 index 000000000..cfd64bbbe --- /dev/null +++ b/src/arrow/js/index.ts @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +export * from './src/Arrow.node';
\ No newline at end of file diff --git a/src/arrow/js/jest.config.js b/src/arrow/js/jest.config.js new file mode 100644 index 000000000..fb3f97c44 --- /dev/null +++ b/src/arrow/js/jest.config.js @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + verbose: false, + testEnvironment: "node", + globals: { + "ts-jest": { + diagnostics: false, + tsconfig: "test/tsconfig.json", + useESM: true, + }, + }, + rootDir: ".", + roots: ["<rootDir>/test/"], + preset: "ts-jest/presets/default-esm", + moduleFileExtensions: ["mjs", "js", "ts"], + coverageReporters: ["lcov", "json"], + coveragePathIgnorePatterns: [ + "fb\\/(File|Message|Schema|Tensor)\\.(js|ts)$", + "test\\/.*\\.(ts|js)$", + "/node_modules/", + ], + transform: { + "^.+\\.js$": "ts-jest", + "^.+\\.ts$": "ts-jest", + }, + transformIgnorePatterns: [ + "/targets/(es5|es2015|esnext|apache-arrow)/", + "/node_modules/(?!@openpgp/web-stream-tools)/", + ], + testRegex: "(.*(-|\\.)(test|spec)s?)\\.(ts|js)$", + testMatch: null, + moduleNameMapper: { + "^apache-arrow$": "<rootDir>/src/Arrow.node", + "^apache-arrow(.*)": "<rootDir>/src$1", + flatbuffers: "flatbuffers/js/flatbuffers.mjs", + }, +}; diff --git a/src/arrow/js/jestconfigs/jest.apache-arrow.config.js b/src/arrow/js/jestconfigs/jest.apache-arrow.config.js new file mode 100644 index 000000000..103dc5a92 --- /dev/null +++ b/src/arrow/js/jestconfigs/jest.apache-arrow.config.js @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + ...require("../jest.config"), + rootDir: "../", + preset: "ts-jest", + moduleFileExtensions: ["js", "ts"], + globals: { + "ts-jest": { + diagnostics: false, + tsconfig: "<rootDir>/test/tsconfig/tsconfig.apache-arrow.json", + }, + }, + moduleNameMapper: { + "^apache-arrow(.*)": "<rootDir>/targets/apache-arrow$1", + }, +}; diff --git a/src/arrow/js/jestconfigs/jest.coverage.config.js b/src/arrow/js/jestconfigs/jest.coverage.config.js new file mode 100644 index 000000000..3b0b6a1c6 --- /dev/null +++ b/src/arrow/js/jestconfigs/jest.coverage.config.js @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + ...require("../jest.config"), + rootDir: "../", + collectCoverage: true, + reporters: undefined, + globals: { + "ts-jest": { + diagnostics: false, + tsconfig: "<rootDir>/test/tsconfig/tsconfig.coverage.json", + useESM: true, + }, + }, +}; diff --git a/src/arrow/js/jestconfigs/jest.es2015.cjs.config.js b/src/arrow/js/jestconfigs/jest.es2015.cjs.config.js new file mode 100644 index 000000000..1d5676761 --- /dev/null +++ b/src/arrow/js/jestconfigs/jest.es2015.cjs.config.js @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + ...require("../jest.config"), + rootDir: "../", + preset: "ts-jest", + moduleFileExtensions: ["js", "ts"], + globals: { + "ts-jest": { + diagnostics: false, + tsconfig: "<rootDir>/test/tsconfig/tsconfig.es2015.cjs.json", + }, + }, + moduleNameMapper: { + "^apache-arrow(.*)": "<rootDir>/targets/es2015/cjs$1", + }, +}; diff --git a/src/arrow/js/jestconfigs/jest.es2015.esm.config.js b/src/arrow/js/jestconfigs/jest.es2015.esm.config.js new file mode 100644 index 000000000..cf564fb23 --- /dev/null +++ b/src/arrow/js/jestconfigs/jest.es2015.esm.config.js @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + ...require("../jest.config"), + rootDir: "../", + globals: { + "ts-jest": { + diagnostics: false, + tsconfig: "<rootDir>/test/tsconfig/tsconfig.es2015.esm.json", + useESM: true, + }, + }, + moduleNameMapper: { + "^apache-arrow(.*)": "<rootDir>/targets/es2015/esm$1", + tslib: "tslib/tslib.es6.js" + }, +}; diff --git a/src/arrow/js/jestconfigs/jest.es2015.umd.config.js b/src/arrow/js/jestconfigs/jest.es2015.umd.config.js new file mode 100644 index 000000000..21f27872d --- /dev/null +++ b/src/arrow/js/jestconfigs/jest.es2015.umd.config.js @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + ...require("../jest.config"), + rootDir: "../", + preset: "ts-jest", + moduleFileExtensions: ["js", "ts"], + globals: { + "ts-jest": { + diagnostics: false, + tsconfig: "<rootDir>/test/tsconfig/tsconfig.es2015.umd.json", + }, + }, + moduleNameMapper: { + "^apache-arrow(.*)": "<rootDir>/targets/es2015/umd/Arrow.js", + }, +}; diff --git a/src/arrow/js/jestconfigs/jest.es5.cjs.config.js b/src/arrow/js/jestconfigs/jest.es5.cjs.config.js new file mode 100644 index 000000000..ae3e9bb42 --- /dev/null +++ b/src/arrow/js/jestconfigs/jest.es5.cjs.config.js @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + ...require("../jest.config"), + rootDir: "../", + preset: "ts-jest", + moduleFileExtensions: ["js", "ts"], + globals: { + "ts-jest": { + diagnostics: false, + tsconfig: "<rootDir>/test/tsconfig/tsconfig.es5.cjs.json", + }, + }, + moduleNameMapper: { + "^apache-arrow(.*)": "<rootDir>/targets/es5/cjs$1", + }, +}; diff --git a/src/arrow/js/jestconfigs/jest.es5.esm.config.js b/src/arrow/js/jestconfigs/jest.es5.esm.config.js new file mode 100644 index 000000000..0a0a21b76 --- /dev/null +++ b/src/arrow/js/jestconfigs/jest.es5.esm.config.js @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + ...require("../jest.config"), + rootDir: "../", + globals: { + "ts-jest": { + diagnostics: false, + tsconfig: "<rootDir>/test/tsconfig/tsconfig.es5.esm.json", + useESM: true, + }, + }, + moduleNameMapper: { + "^apache-arrow(.*)": "<rootDir>/targets/es5/esm$1", + tslib: "tslib/tslib.es6.js" + }, +}; diff --git a/src/arrow/js/jestconfigs/jest.es5.umd.config.js b/src/arrow/js/jestconfigs/jest.es5.umd.config.js new file mode 100644 index 000000000..f52af07bc --- /dev/null +++ b/src/arrow/js/jestconfigs/jest.es5.umd.config.js @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + ...require("../jest.config"), + rootDir: "../", + preset: "ts-jest", + moduleFileExtensions: ["js", "ts"], + globals: { + "ts-jest": { + diagnostics: false, + tsconfig: "<rootDir>/test/tsconfig/tsconfig.es5.umd.json", + }, + }, + moduleNameMapper: { + "^apache-arrow(.*)": "<rootDir>/targets/es5/umd/Arrow.js", + }, +}; diff --git a/src/arrow/js/jestconfigs/jest.esnext.cjs.config.js b/src/arrow/js/jestconfigs/jest.esnext.cjs.config.js new file mode 100644 index 000000000..8be999e3d --- /dev/null +++ b/src/arrow/js/jestconfigs/jest.esnext.cjs.config.js @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + ...require("../jest.config"), + rootDir: "../", + preset: "ts-jest", + moduleFileExtensions: ["js", "ts"], + globals: { + "ts-jest": { + diagnostics: false, + tsconfig: "<rootDir>/test/tsconfig/tsconfig.esnext.cjs.json", + }, + }, + moduleNameMapper: { + "^apache-arrow(.*)": "<rootDir>/targets/esnext/cjs$1", + }, +}; diff --git a/src/arrow/js/jestconfigs/jest.esnext.esm.config.js b/src/arrow/js/jestconfigs/jest.esnext.esm.config.js new file mode 100644 index 000000000..aca4c5208 --- /dev/null +++ b/src/arrow/js/jestconfigs/jest.esnext.esm.config.js @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + ...require("../jest.config"), + rootDir: "../", + globals: { + "ts-jest": { + diagnostics: false, + tsconfig: "<rootDir>/test/tsconfig/tsconfig.esnext.esm.json", + useESM: true, + }, + }, + moduleNameMapper: { + "^apache-arrow(.*)": "<rootDir>/targets/esnext/esm$1", + tslib: "tslib/tslib.es6.js" + }, +}; diff --git a/src/arrow/js/jestconfigs/jest.esnext.umd.config.js b/src/arrow/js/jestconfigs/jest.esnext.umd.config.js new file mode 100644 index 000000000..5013d45e0 --- /dev/null +++ b/src/arrow/js/jestconfigs/jest.esnext.umd.config.js @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + ...require("../jest.config"), + rootDir: "../", + preset: "ts-jest", + moduleFileExtensions: ["js", "ts"], + globals: { + "ts-jest": { + diagnostics: false, + tsconfig: "<rootDir>/test/tsconfig/tsconfig.esnext.umd.json", + }, + }, + moduleNameMapper: { + "^apache-arrow(.*)": "<rootDir>/targets/esnext/umd/Arrow.js", + }, +}; diff --git a/src/arrow/js/jestconfigs/jest.src.config.js b/src/arrow/js/jestconfigs/jest.src.config.js new file mode 100644 index 000000000..08ccad061 --- /dev/null +++ b/src/arrow/js/jestconfigs/jest.src.config.js @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + ...require("../jest.config"), + rootDir: "../", + globals: { + "ts-jest": { + diagnostics: false, + tsconfig: "<rootDir>/test/tsconfig/tsconfig.src.json", + useESM: true, + }, + }, +}; diff --git a/src/arrow/js/jestconfigs/jest.ts.config.js b/src/arrow/js/jestconfigs/jest.ts.config.js new file mode 100644 index 000000000..e56161b8b --- /dev/null +++ b/src/arrow/js/jestconfigs/jest.ts.config.js @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + ...require("../jest.config"), + rootDir: "../", + globals: { + "ts-jest": { + diagnostics: false, + tsconfig: "<rootDir>/test/tsconfig/tsconfig.ts.json", + useESM: true, + }, + }, + moduleNameMapper: { + "^apache-arrow(.*)": "<rootDir>/targets/ts$1" + }, +}; diff --git a/src/arrow/js/lerna.json b/src/arrow/js/lerna.json new file mode 100644 index 000000000..99d6f64fb --- /dev/null +++ b/src/arrow/js/lerna.json @@ -0,0 +1,12 @@ +{ + "lerna": "3.22.1", + "version": "4.0.0", + "npmClient": "yarn", + "packages": [ + "targets/ts", + "targets/es5/*", + "targets/es2015/*", + "targets/esnext/*", + "targets/apache-arrow" + ] +} diff --git a/src/arrow/js/npm-release.sh b/src/arrow/js/npm-release.sh new file mode 100755 index 000000000..5d92d01cb --- /dev/null +++ b/src/arrow/js/npm-release.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +set -e + +# validate the targets pass all tests before publishing +yarn --frozen-lockfile +yarn gulp + +read -p "Please enter your npm 2FA one-time password (or leave empty if you don't have 2FA enabled): " NPM_OTP </dev/tty + +# publish the JS target modules to npm +yarn lerna exec --concurrency 1 --no-bail "npm publish${NPM_OTP:+ --otp=$NPM_OTP}" diff --git a/src/arrow/js/package.json b/src/arrow/js/package.json new file mode 100644 index 000000000..2c4160f58 --- /dev/null +++ b/src/arrow/js/package.json @@ -0,0 +1,105 @@ +{ + "name": "apache-arrow", + "description": "Apache Arrow columnar in-memory format", + "bin": { + "arrow2csv": "bin/arrow2csv.js" + }, + "scripts": { + "lerna": "lerna", + "test": "cross-env NODE_NO_WARNINGS=1 gulp test", + "build": "cross-env NODE_NO_WARNINGS=1 gulp build", + "clean": "cross-env NODE_NO_WARNINGS=1 gulp clean", + "debug": "cross-env NODE_NO_WARNINGS=1 gulp debug", + "perf": "ts-node-transpile-only ./perf/index.ts", + "test:integration": "node ./bin/integration.js --mode validate", + "release": "./npm-release.sh", + "clean:all": "run-p clean clean:testdata", + "clean:testdata": "gulp clean:testdata", + "create:testdata": "gulp create:testdata", + "test:coverage": "gulp test -t src --coverage", + "doc": "del-cli ./doc && typedoc --options typedoc.js", + "lint": "eslint src test --fix", + "lint:ci": "eslint src test", + "prepublishOnly": "echo \"Error: do 'yarn release' instead of 'npm publish'\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/apache/arrow.git" + }, + "keywords": [ + "apache", + "arrow" + ], + "author": "Apache Software Foundation", + "license": "Apache-2.0", + "bugs": { + "url": "https://issues.apache.org/jira/projects/ARROW" + }, + "homepage": "https://github.com/apache/arrow/blob/master/js/README.md", + "files": [ + "bin", + "src", + "gulp", + "jestconfigs", + "test", + "*.json", + "tsconfigs", + "README.md", + "gulpfile.js", + "npm-release.sh", + "jest.config.js" + ], + "dependencies": { + "@types/flatbuffers": "^1.10.0", + "@types/node": "^16.4.0", + "command-line-args": "5.1.3", + "command-line-usage": "6.1.1", + "flatbuffers": "1.12.0", + "json-bignum": "^0.0.3", + "pad-left": "^2.1.0", + "tslib": "^2.3.0" + }, + "devDependencies": { + "@openpgp/web-stream-tools": "0.0.6", + "@types/glob": "7.1.4", + "@types/jest": "26.0.24", + "@types/randomatic": "3.1.2", + "@typescript-eslint/eslint-plugin": "4.28.4", + "@typescript-eslint/parser": "4.28.4", + "async-done": "1.3.2", + "benny": "3.6.15", + "cpy": "8.1.2", + "cross-env": "7.0.3", + "del-cli": "4.0.1", + "eslint": "7.31.0", + "eslint-plugin-jest": "24.3.7", + "esm": "https://github.com/jsg2021/esm/releases/download/v3.x.x-pr883/esm-3.x.x-pr883.tgz", + "glob": "7.1.7", + "google-closure-compiler": "20210601.0.0", + "gulp": "4.0.2", + "gulp-json-transform": "0.4.7", + "gulp-rename": "2.0.0", + "gulp-sourcemaps": "3.0.0", + "gulp-typescript": "5.0.1", + "ix": "4.4.1", + "jest": "27.0.6", + "jest-silent-reporter": "0.5.0", + "lerna": "4.0.0", + "memfs": "3.2.2", + "mkdirp": "1.0.4", + "multistream": "4.1.0", + "npm-run-all": "4.1.5", + "randomatic": "3.1.1", + "rxjs": "7.2.0", + "ts-jest": "27.0.3", + "ts-node": "10.1.0", + "typedoc": "0.21.4", + "typescript": "4.0.2", + "web-streams-polyfill": "3.0.3", + "xml2js": "0.4.23" + }, + "engines": { + "node": ">=12.0" + }, + "version": "6.0.1" +} diff --git a/src/arrow/js/perf/config.ts b/src/arrow/js/perf/config.ts new file mode 100644 index 000000000..08ea9ecc1 --- /dev/null +++ b/src/arrow/js/perf/config.ts @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import * as Arrow from '../src/Arrow.dom'; + +// from https://stackoverflow.com/a/19303725/214950 +let seed = 1; +function random() { + const x = Math.sin(seed++) * 10000; + return x - Math.floor(x); +} + +console.time('Prepare Data'); + +const LENGTH = 100000; +const NUM_BATCHES = 10; + +const values = Arrow.Utf8Vector.from(['Charlottesville', 'New York', 'San Francisco', 'Seattle', 'Terre Haute', 'Washington, DC']); + +const batches = Array.from({length: NUM_BATCHES}).map(() => { + const lat = Float32Array.from( + { length: LENGTH }, + () => ((random() - 0.5) * 2 * 90)); + const lng = Float32Array.from( + { length: LENGTH }, + () => ((random() - 0.5) * 2 * 90)); + + const origin = Uint8Array.from( + { length: LENGTH }, + () => (random() * 6)); + const destination = Uint8Array.from( + { length: LENGTH }, + () => (random() * 6)); + + const originType = new Arrow.Dictionary(values.type, new Arrow.Int8, 0, false); + const destinationType = new Arrow.Dictionary(values.type, new Arrow.Int8, 0, false); + + return Arrow.RecordBatch.new({ + 'lat': Arrow.Float32Vector.from(lat), + 'lng': Arrow.Float32Vector.from(lng), + 'origin': Arrow.Vector.new(Arrow.Data.Dictionary(originType, 0, origin.length, 0, null, origin, values)), + 'destination': Arrow.Vector.new(Arrow.Data.Dictionary(destinationType, 0, destination.length, 0, null, destination, values)), + }); +}); + +const tracks = new Arrow.DataFrame(batches[0].schema, batches); + +console.timeEnd('Prepare Data'); + +export default [ + { + name: 'tracks', + df: tracks, + ipc: tracks.serialize(), + countBys: ['origin', 'destination'], + counts: [ + {column: 'lat', test: 'gt' as 'gt' | 'eq', value: 0 }, + {column: 'lng', test: 'gt' as 'gt' | 'eq', value: 0 }, + {column: 'origin', test: 'eq' as 'gt' | 'eq', value: 'Seattle'}, + ], + } +]; diff --git a/src/arrow/js/perf/index.ts b/src/arrow/js/perf/index.ts new file mode 100644 index 000000000..9f6cb8f79 --- /dev/null +++ b/src/arrow/js/perf/index.ts @@ -0,0 +1,234 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Alternatively, use bundles for performance tests +// import * as Arrow from '../targets/es5/umd'; +// import * as Arrow from '../targets/es5/cjs'; +// import * as Arrow from '../targets/es2015/umd'; +// import * as Arrow from '../targets/es2015/cjs'; + +import * as Arrow from '../src/Arrow'; + +import config from './config'; +import b from 'benny'; +import { CaseResult, Summary } from 'benny/lib/internal/common-types'; +import kleur from 'kleur'; + +const { predicate, Table, RecordBatchReader } = Arrow; +const { col } = predicate; + + +const args = process.argv.slice(2); +const json = args[0] === '--json'; + +const formatter = new Intl.NumberFormat(); +function formatNumber(number: number, precision = 0) { + const rounded = number > precision * 10 ? Math.round(number) : parseFloat((number).toPrecision(precision)); + return formatter.format(rounded); +} + +const results: CaseResult[] = []; + +function cycle(result: CaseResult, _summary: Summary) { + const duration = result.details.median * 1000; + if (json) { + result.suite = _summary.name; + results.push(result); + } + console.log( + `${kleur.cyan(result.name)} ${formatNumber(result.ops, 3)} ops/s ±${result.margin.toPrecision(2)}%, ${formatNumber(duration, 2)} ms, ${kleur.gray(result.samples + ' samples')}`, + ); +} + +for (const { name, ipc, df } of config) { + b.suite( + `Parse`, + + b.add(`dataset: ${name}, function: Table.from`, () => { + Table.from(ipc); + }), + + b.add(`dataset: ${name}, function: readBatches`, () => { + for (const _recordBatch of RecordBatchReader.from(ipc)) {} + }), + + b.add(`dataset: ${name}, function: serialize`, () => { + df.serialize(); + }), + + b.cycle(cycle) + ); + + const schema = df.schema; + + const suites = [{ + suite_name: `Get values by index`, + fn(vector: Arrow.Column<any>) { + for (let i = -1, n = vector.length; ++i < n;) { + vector.get(i); + } + } + }, { + suite_name: `Iterate vectors`, + fn(vector: Arrow.Column<any>) { for (const _value of vector) {} } + }, { + suite_name: `Slice toArray vectors`, + fn(vector: Arrow.Column<any>) { vector.slice().toArray(); } + }, { + suite_name: `Slice vectors`, + fn(vector: Arrow.Column<any>) { vector.slice(); } + }]; + + for (const {suite_name, fn} of suites) { + b.suite( + suite_name, + + ...schema.fields.map((f, i) => { + const vector = df.getColumnAt(i)!; + return b.add(`dataset: ${name}, column: ${f.name}, length: ${formatNumber(vector.length)}, type: ${vector.type}`, () => { + fn(vector); + }); + }), + + b.cycle(cycle) + ); + } +} + + +for (const { name, df, countBys, counts } of config) { + b.suite( + `DataFrame Iterate`, + + b.add(`dataset: ${name}, length: ${formatNumber(df.length)}`, () => { + for (const _value of df) {} + }), + + b.cycle(cycle) + ); + + b.suite( + `DataFrame Count By`, + + ...countBys.map((column: string) => b.add( + `dataset: ${name}, column: ${column}, length: ${formatNumber(df.length)}, type: ${df.schema.fields.find((c)=> c.name === column)!.type}`, + () => df.countBy(column) + )), + + b.cycle(cycle) + ); + + b.suite( + `DataFrame Filter-Scan Count`, + + ...counts.map(({ column, test, value }: {column: string; test: 'gt' | 'eq'; value: number | string}) => b.add( + `dataset: ${name}, column: ${column}, length: ${formatNumber(df.length)}, type: ${df.schema.fields.find((c)=> c.name === column)!.type}, test: ${test}, value: ${value}`, + () => { + let filteredDf: Arrow.FilteredDataFrame; + if (test == 'gt') { + filteredDf = df.filter(col(column).gt(value)); + } else if (test == 'eq') { + filteredDf = df.filter(col(column).eq(value)); + } else { + throw new Error(`Unrecognized test "${test}"`); + } + + return () => filteredDf.count(); + } + )), + + b.cycle(cycle) + ); + + b.suite( + `DataFrame Filter-Iterate`, + + ...counts.map(({ column, test, value }: {column: string; test: 'gt' | 'eq'; value: number | string}) => b.add( + `dataset: ${name}, column: ${column}, length: ${formatNumber(df.length)}, type: ${df.schema.fields.find((c)=> c.name === column)!.type}, test: ${test}, value: ${value}`, + () => { + let filteredDf: Arrow.FilteredDataFrame; + if (test == 'gt') { + filteredDf = df.filter(col(column).gt(value)); + } else if (test == 'eq') { + filteredDf = df.filter(col(column).eq(value)); + } else { + throw new Error(`Unrecognized test "${test}"`); + } + + return () => { + for (const _value of filteredDf) {} + }; + } + )), + + b.cycle(cycle) + ); + + b.suite( + `DataFrame Direct Count`, + + ...counts.map(({ column, test, value }: {column: string; test: 'gt' | 'eq'; value: number | string}) => b.add( + `dataset: ${name}, column: ${column}, length: ${formatNumber(df.length)}, type: ${df.schema.fields.find((c)=> c.name === column)!.type}, test: ${test}, value: ${value}`, + () => { + const colidx = df.schema.fields.findIndex((c)=> c.name === column); + + if (test == 'gt') { + return () => { + let sum = 0; + const batches = df.chunks; + const numBatches = batches.length; + for (let batchIndex = -1; ++batchIndex < numBatches;) { + // load batches + const batch = batches[batchIndex]; + const vector = batch.getChildAt(colidx)!; + // yield all indices + for (let index = -1, length = batch.length; ++index < length;) { + sum += (vector.get(index) >= value) ? 1 : 0; + } + } + return sum; + }; + } else if (test == 'eq') { + return () => { + let sum = 0; + const batches = df.chunks; + const numBatches = batches.length; + for (let batchIndex = -1; ++batchIndex < numBatches;) { + // load batches + const batch = batches[batchIndex]; + const vector = batch.getChildAt(colidx)!; + // yield all indices + for (let index = -1, length = batch.length; ++index < length;) { + sum += (vector.get(index) === value) ? 1 : 0; + } + } + return sum; + }; + } else { + throw new Error(`Unrecognized test "${test}"`); + } + } + )), + + b.cycle(cycle), + + b.complete(() => { + // last benchmark finished + json && process.stderr.write(JSON.stringify(results, null, 2)); + }) + ); +} diff --git a/src/arrow/js/src/Arrow.dom.ts b/src/arrow/js/src/Arrow.dom.ts new file mode 100644 index 000000000..07f0c8b8e --- /dev/null +++ b/src/arrow/js/src/Arrow.dom.ts @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import streamAdapters from './io/adapters'; +import { Builder } from './builder/index'; +import { RecordBatchReader, RecordBatchFileReader, RecordBatchStreamReader, } from './ipc/reader'; +import { RecordBatchWriter, RecordBatchFileWriter, RecordBatchStreamWriter, } from './ipc/writer'; +import { toDOMStream } from './io/whatwg/iterable'; +import { builderThroughDOMStream } from './io/whatwg/builder'; +import { recordBatchReaderThroughDOMStream } from './io/whatwg/reader'; +import { recordBatchWriterThroughDOMStream } from './io/whatwg/writer'; + +streamAdapters.toDOMStream = toDOMStream; +Builder['throughDOM'] = builderThroughDOMStream; +RecordBatchReader['throughDOM'] = recordBatchReaderThroughDOMStream; +RecordBatchFileReader['throughDOM'] = recordBatchReaderThroughDOMStream; +RecordBatchStreamReader['throughDOM'] = recordBatchReaderThroughDOMStream; +RecordBatchWriter['throughDOM'] = recordBatchWriterThroughDOMStream; +RecordBatchFileWriter['throughDOM'] = recordBatchWriterThroughDOMStream; +RecordBatchStreamWriter['throughDOM'] = recordBatchWriterThroughDOMStream; + +export { + DateUnit, IntervalUnit, MessageHeader, MetadataVersion, Precision, TimeUnit, Type, UnionMode, BufferType, + Data, + DataType, + Null, + Bool, + Int, Int8, Int16, Int32, Int64, Uint8, Uint16, Uint32, Uint64, + Float, Float16, Float32, Float64, + Utf8, + Binary, + FixedSizeBinary, + Date_, DateDay, DateMillisecond, + Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, + Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, + Decimal, + List, + Struct, + Union, DenseUnion, SparseUnion, + Dictionary, + Interval, IntervalDayTime, IntervalYearMonth, + FixedSizeList, + Map_, + Table, + Column, + Schema, Field, + Visitor, + Vector, + BaseVector, + BinaryVector, + BoolVector, + Chunked, + DateVector, DateDayVector, DateMillisecondVector, + DecimalVector, + DictionaryVector, + FixedSizeBinaryVector, + FixedSizeListVector, + FloatVector, Float16Vector, Float32Vector, Float64Vector, + IntervalVector, IntervalDayTimeVector, IntervalYearMonthVector, + IntVector, Int8Vector, Int16Vector, Int32Vector, Int64Vector, Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector, + ListVector, + MapVector, + NullVector, + StructVector, + TimestampVector, TimestampSecondVector, TimestampMillisecondVector, TimestampMicrosecondVector, TimestampNanosecondVector, + TimeVector, TimeSecondVector, TimeMillisecondVector, TimeMicrosecondVector, TimeNanosecondVector, + UnionVector, DenseUnionVector, SparseUnionVector, + Utf8Vector, + ByteStream, AsyncByteStream, AsyncByteQueue, ReadableSource, WritableSink, + RecordBatchReader, RecordBatchFileReader, RecordBatchStreamReader, AsyncRecordBatchFileReader, AsyncRecordBatchStreamReader, + RecordBatchWriter, RecordBatchFileWriter, RecordBatchStreamWriter, RecordBatchJSONWriter, + MessageReader, AsyncMessageReader, JSONMessageReader, + Message, + RecordBatch, + ArrowJSONLike, FileHandle, Readable, Writable, ReadableWritable, ReadableDOMStreamOptions, + DataFrame, FilteredDataFrame, CountByResult, BindFunc, NextFunc, + predicate, + util, + Builder, + BinaryBuilder, + BoolBuilder, + DateBuilder, DateDayBuilder, DateMillisecondBuilder, + DecimalBuilder, + DictionaryBuilder, + FixedSizeBinaryBuilder, + FixedSizeListBuilder, + FloatBuilder, Float16Builder, Float32Builder, Float64Builder, + IntervalBuilder, IntervalDayTimeBuilder, IntervalYearMonthBuilder, + IntBuilder, Int8Builder, Int16Builder, Int32Builder, Int64Builder, Uint8Builder, Uint16Builder, Uint32Builder, Uint64Builder, + ListBuilder, + MapBuilder, + NullBuilder, + StructBuilder, + TimestampBuilder, TimestampSecondBuilder, TimestampMillisecondBuilder, TimestampMicrosecondBuilder, TimestampNanosecondBuilder, + TimeBuilder, TimeSecondBuilder, TimeMillisecondBuilder, TimeMicrosecondBuilder, TimeNanosecondBuilder, + UnionBuilder, DenseUnionBuilder, SparseUnionBuilder, + Utf8Builder, + isTypedArray, +} from './Arrow'; diff --git a/src/arrow/js/src/Arrow.node.ts b/src/arrow/js/src/Arrow.node.ts new file mode 100644 index 000000000..44221f613 --- /dev/null +++ b/src/arrow/js/src/Arrow.node.ts @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import streamAdapters from './io/adapters'; +import { Builder } from './builder/index'; +import { RecordBatchReader } from './ipc/reader'; +import { RecordBatchWriter } from './ipc/writer'; +import { toNodeStream } from './io/node/iterable'; +import { builderThroughNodeStream } from './io/node/builder'; +import { recordBatchReaderThroughNodeStream } from './io/node/reader'; +import { recordBatchWriterThroughNodeStream } from './io/node/writer'; + +streamAdapters.toNodeStream = toNodeStream; +Builder['throughNode'] = builderThroughNodeStream; +RecordBatchReader['throughNode'] = recordBatchReaderThroughNodeStream; +RecordBatchWriter['throughNode'] = recordBatchWriterThroughNodeStream; + +export * from './Arrow.dom'; diff --git a/src/arrow/js/src/Arrow.ts b/src/arrow/js/src/Arrow.ts new file mode 100644 index 000000000..8bf296310 --- /dev/null +++ b/src/arrow/js/src/Arrow.ts @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +export { + DateUnit, + TimeUnit, + Precision, + UnionMode, + IntervalUnit, + MetadataVersion, +} from './fb/Schema'; + +export { MessageHeader } from './fb/Message'; + +export { Type, BufferType } from './enum'; + +export { Data } from './data'; +export { + DataType, + Null, + Bool, + Int, Int8, Int16, Int32, Int64, Uint8, Uint16, Uint32, Uint64, + Float, Float16, Float32, Float64, + Utf8, + Binary, + FixedSizeBinary, + Date_, DateDay, DateMillisecond, + Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, + Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, + Decimal, + List, + Struct, + Union, DenseUnion, SparseUnion, + Dictionary, + Interval, IntervalDayTime, IntervalYearMonth, + FixedSizeList, + Map_, +} from './type'; + +export { Table } from './table'; +export { Column } from './column'; +export { Visitor } from './visitor'; +export { Schema, Field } from './schema'; +export { + Vector, + BaseVector, + BinaryVector, + BoolVector, + Chunked, + DateVector, DateDayVector, DateMillisecondVector, + DecimalVector, + DictionaryVector, + FixedSizeBinaryVector, + FixedSizeListVector, + FloatVector, Float16Vector, Float32Vector, Float64Vector, + IntervalVector, IntervalDayTimeVector, IntervalYearMonthVector, + IntVector, Int8Vector, Int16Vector, Int32Vector, Int64Vector, Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector, + ListVector, + MapVector, + NullVector, + StructVector, + TimestampVector, TimestampSecondVector, TimestampMillisecondVector, TimestampMicrosecondVector, TimestampNanosecondVector, + TimeVector, TimeSecondVector, TimeMillisecondVector, TimeMicrosecondVector, TimeNanosecondVector, + UnionVector, DenseUnionVector, SparseUnionVector, + Utf8Vector, +} from './vector/index'; + +export { + Builder, + BinaryBuilder, + BoolBuilder, + DateBuilder, DateDayBuilder, DateMillisecondBuilder, + DecimalBuilder, + DictionaryBuilder, + FixedSizeBinaryBuilder, + FixedSizeListBuilder, + FloatBuilder, Float16Builder, Float32Builder, Float64Builder, + IntervalBuilder, IntervalDayTimeBuilder, IntervalYearMonthBuilder, + IntBuilder, Int8Builder, Int16Builder, Int32Builder, Int64Builder, Uint8Builder, Uint16Builder, Uint32Builder, Uint64Builder, + ListBuilder, + MapBuilder, + NullBuilder, + StructBuilder, + TimestampBuilder, TimestampSecondBuilder, TimestampMillisecondBuilder, TimestampMicrosecondBuilder, TimestampNanosecondBuilder, + TimeBuilder, TimeSecondBuilder, TimeMillisecondBuilder, TimeMicrosecondBuilder, TimeNanosecondBuilder, + UnionBuilder, DenseUnionBuilder, SparseUnionBuilder, + Utf8Builder, +} from './builder/index'; + +export { ByteStream, AsyncByteStream, AsyncByteQueue, ReadableSource, WritableSink } from './io/stream'; +export { RecordBatchReader, RecordBatchFileReader, RecordBatchStreamReader, AsyncRecordBatchFileReader, AsyncRecordBatchStreamReader } from './ipc/reader'; +export { RecordBatchWriter, RecordBatchFileWriter, RecordBatchStreamWriter, RecordBatchJSONWriter } from './ipc/writer'; +export { MessageReader, AsyncMessageReader, JSONMessageReader } from './ipc/message'; +export { Message } from './ipc/metadata/message'; +export { RecordBatch } from './recordbatch'; +export { ArrowJSONLike, FileHandle, Readable, Writable, ReadableWritable, ReadableDOMStreamOptions } from './io/interfaces'; +export { DataFrame, FilteredDataFrame, CountByResult, BindFunc, NextFunc } from './compute/dataframe'; + +import * as util_bn_ from './util/bn'; +import * as util_int_ from './util/int'; +import * as util_bit_ from './util/bit'; +import * as util_math_ from './util/math'; +import * as util_buffer_ from './util/buffer'; +import * as util_vector_ from './util/vector'; +import * as predicate from './compute/predicate'; +import { compareSchemas, compareFields, compareTypes } from './visitor/typecomparator'; + +export { predicate }; +/** @ignore */ +export const util = { + ...util_bn_, + ...util_int_, + ...util_bit_, + ...util_math_, + ...util_buffer_, + ...util_vector_, + compareSchemas, + compareFields, + compareTypes, +}; + +export { isTypedArray } from './util/args'; diff --git a/src/arrow/js/src/bin/arrow2csv.ts b/src/arrow/js/src/bin/arrow2csv.ts new file mode 100644 index 000000000..d5803cce0 --- /dev/null +++ b/src/arrow/js/src/bin/arrow2csv.ts @@ -0,0 +1,334 @@ +#! /usr/bin/env node + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import * as fs from 'fs'; +import * as stream from 'stream'; +import { valueToString } from '../util/pretty'; +import { Schema, RecordBatch, RecordBatchReader, AsyncByteQueue } from '../Arrow.node'; + +/* eslint-disable @typescript-eslint/no-require-imports */ + +const padLeft = require('pad-left'); +const bignumJSONParse = require('json-bignum').parse; +const argv = require(`command-line-args`)(cliOpts(), { partial: true }); +const files = argv.help ? [] : [...(argv.file || []), ...(argv._unknown || [])].filter(Boolean); + +const state = { ...argv, closed: false, maxColWidths: [10] }; + +type ToStringState = { + hr: string; + sep: string; + schema: any; + closed: boolean; + metadata: boolean; + maxColWidths: number[]; +}; + +(async () => { + + const sources = argv.help ? [] : [ + ...files.map((file) => () => fs.createReadStream(file)), + ...(process.stdin.isTTY ? [] : [() => process.stdin]) + ].filter(Boolean) as (() => NodeJS.ReadableStream)[]; + + let reader: RecordBatchReader | null; + let hasReaders = false; + + for (const source of sources) { + if (state.closed) { break; } + for await (reader of recordBatchReaders(source)) { + hasReaders = true; + const transformToString = batchesToString(state, reader.schema); + await pipeTo( + reader.pipe(transformToString), + process.stdout, { end: false } + ).catch(() => state.closed = true); // Handle EPIPE errors + } + if (state.closed) { break; } + } + + return hasReaders ? 0 : print_usage(); +})() +.then((x) => +x || 0, (err) => { + if (err) { + console.error(`${err?.stack || err}`); + } + return process.exitCode || 1; +}).then((code) => process.exit(code)); + +function pipeTo(source: NodeJS.ReadableStream, sink: NodeJS.WritableStream, opts?: { end: boolean }) { + return new Promise((resolve, reject) => { + + source.on('end', onEnd).pipe(sink, opts).on('error', onErr); + + function onEnd() { done(undefined, resolve); } + function onErr(err: any) { done(err, reject); } + function done(e: any, cb: (e?: any) => void) { + source.removeListener('end', onEnd); + sink.removeListener('error', onErr); + cb(e); + } + }); +} + +async function *recordBatchReaders(createSourceStream: () => NodeJS.ReadableStream) { + + const json = new AsyncByteQueue(); + const stream = new AsyncByteQueue(); + const source = createSourceStream(); + let reader: RecordBatchReader | null = null; + let readers: AsyncIterable<RecordBatchReader> | null = null; + // tee the input source, just in case it's JSON + source.on('end', () => [stream, json].forEach((y) => y.close())) + .on('data', (x) => [stream, json].forEach((y) => y.write(x))) + .on('error', (e) => [stream, json].forEach((y) => y.abort(e))); + + try { + for await (reader of RecordBatchReader.readAll(stream)) { + reader && (yield reader); + } + if (reader) return; + } catch (e) { readers = null; } + + if (!readers) { + await json.closed; + if (source instanceof fs.ReadStream) { source.close(); } + // If the data in the `json` ByteQueue parses to JSON, then assume it's Arrow JSON from a file or stdin + try { + for await (reader of RecordBatchReader.readAll(bignumJSONParse(await json.toString()))) { + reader && (yield reader); + } + } catch (e) { readers = null; } + } +} + +function batchesToString(state: ToStringState, schema: Schema) { + + let rowId = 0; + let batchId = -1; + let maxColWidths = [10]; + const { hr, sep } = state; + + const header = ['row_id', ...schema.fields.map((f) => `${f}`)].map(valueToString); + + state.maxColWidths = header.map((x, i) => Math.max(maxColWidths[i] || 0, x.length)); + + return new stream.Transform({ + encoding: 'utf8', + writableObjectMode: true, + readableObjectMode: false, + final(cb: (error?: Error | null) => void) { + // if there were no batches, then print the Schema, and metadata + if (batchId === -1) { + hr && this.push(`${horizontalRule(state.maxColWidths, hr, sep)}\n\n`); + this.push(`${formatRow(header, maxColWidths, sep)}\n`); + if (state.metadata && schema.metadata.size > 0) { + this.push(`metadata:\n${formatMetadata(schema.metadata)}\n`); + } + } + hr && this.push(`${horizontalRule(state.maxColWidths, hr, sep)}\n\n`); + cb(); + }, + transform(batch: RecordBatch, _enc: string, cb: (error?: Error, data?: any) => void) { + + batch = !state.schema?.length ? batch : batch.select(...state.schema); + + if (state.closed) { return cb(undefined, null); } + + // Pass one to convert to strings and count max column widths + state.maxColWidths = measureColumnWidths(rowId, batch, header.map((x, i) => Math.max(maxColWidths[i] || 0, x.length))); + + // If this is the first batch in a stream, print a top horizontal rule, schema metadata, and + if (++batchId === 0) { + hr && this.push(`${horizontalRule(state.maxColWidths, hr, sep)}\n`); + if (state.metadata && batch.schema.metadata.size > 0) { + this.push(`metadata:\n${formatMetadata(batch.schema.metadata)}\n`); + hr && this.push(`${horizontalRule(state.maxColWidths, hr, sep)}\n`); + } + if (batch.length <= 0 || batch.numCols <= 0) { + this.push(`${formatRow(header, maxColWidths = state.maxColWidths, sep)}\n`); + } + } + + if (batch.length > 0 && batch.numCols > 0) { + // If any of the column widths changed, print the header again + if (rowId % 350 !== 0 && JSON.stringify(state.maxColWidths) !== JSON.stringify(maxColWidths)) { + this.push(`${formatRow(header, state.maxColWidths, sep)}\n`); + } + maxColWidths = state.maxColWidths; + for (const row of batch) { + if (state.closed) { break; } else if (!row) { continue; } + if (rowId++ % 350 === 0) { + this.push(`${formatRow(header, maxColWidths, sep)}\n`); + } + this.push(`${formatRow([rowId, ...row.toArray()].map(valueToString), maxColWidths, sep)}\n`); + } + } + cb(); + } + }); +} + +function horizontalRule(maxColWidths: number[], hr = '', sep = ' | ') { + return ` ${padLeft('', maxColWidths.reduce((x, y) => x + y, -2 + maxColWidths.length * sep.length), hr)}`; +} + +function formatRow(row: string[] = [], maxColWidths: number[] = [], sep = ' | ') { + return `${row.map((x, j) => padLeft(x, maxColWidths[j])).join(sep)}`; +} + +function formatMetadata(metadata: Map<string, string>) { + + return [...metadata].map(([key, val]) => + ` ${key}: ${formatMetadataValue(val)}` + ).join(', \n'); + + function formatMetadataValue(value = '') { + let parsed = value; + try { + parsed = JSON.stringify(JSON.parse(value), null, 2); + } catch (e) { parsed = value; } + return valueToString(parsed).split('\n').join('\n '); + } +} + +function measureColumnWidths(rowId: number, batch: RecordBatch, maxColWidths: number[] = []) { + let val: any, j = 0; + for (const row of batch) { + if (!row) { continue; } + maxColWidths[j = 0] = Math.max(maxColWidths[0] || 0, (`${rowId++}`).length); + for (val of row) { + if (val && typedArrayElementWidths.has(val.constructor) && (typeof val[Symbol.toPrimitive] !== 'function')) { + // If we're printing a column of TypedArrays, ensure the column is wide enough to accommodate + // the widest possible element for a given byte size, since JS omits leading zeroes. For example: + // 1 | [1137743649,2170567488,244696391,2122556476] + // 2 | null + // 3 | [637174007,2142281880,961736230,2912449282] + // 4 | [1035112265,21832886,412842672,2207710517] + // 5 | null + // 6 | null + // 7 | [2755142991,4192423256,2994359,467878370] + const elementWidth = typedArrayElementWidths.get(val.constructor)!; + + maxColWidths[j + 1] = Math.max(maxColWidths[j + 1] || 0, + 2 + // brackets on each end + (val.length - 1) + // commas between elements + (val.length * elementWidth) // width of stringified 2^N-1 + ); + } else { + maxColWidths[j + 1] = Math.max(maxColWidths[j + 1] || 0, valueToString(val).length); + } + ++j; + } + } + return maxColWidths; +} + +// Measure the stringified representation of 2^N-1 for each TypedArray variant +const typedArrayElementWidths = (() => { + const maxElementWidth = (ArrayType: any) => { + const octets = Array.from({ length: ArrayType.BYTES_PER_ELEMENT - 1 }, _ => 255); + return `${new ArrayType(new Uint8Array([...octets, 254]).buffer)[0]}`.length; + }; + return new Map<any, number>([ + [Int8Array, maxElementWidth(Int8Array)], + [Int16Array, maxElementWidth(Int16Array)], + [Int32Array, maxElementWidth(Int32Array)], + [Uint8Array, maxElementWidth(Uint8Array)], + [Uint16Array, maxElementWidth(Uint16Array)], + [Uint32Array, maxElementWidth(Uint32Array)], + [Float32Array, maxElementWidth(Float32Array)], + [Float64Array, maxElementWidth(Float64Array)], + [Uint8ClampedArray, maxElementWidth(Uint8ClampedArray)] + ]); +})(); + +function cliOpts() { + return [ + { + type: String, + name: 'schema', alias: 's', + optional: true, multiple: true, + typeLabel: '{underline columns}', + description: 'A space-delimited list of column names' + }, + { + type: String, + name: 'file', alias: 'f', + optional: true, multiple: true, + description: 'The Arrow file to read' + }, + { + type: String, + name: 'sep', optional: true, default: ' | ', + description: 'The column separator character (default: " | ")' + }, + { + type: String, + name: 'hr', optional: true, default: '', + description: 'The horizontal border character (default: "")' + }, + { + type: Boolean, + name: 'metadata', alias: 'm', + optional: true, default: false, + description: 'Flag to print Schema metadata (default: false)' + }, + { + type: Boolean, + name: 'help', optional: true, default: false, + description: 'Print this usage guide.' + } + ]; +} + +function print_usage() { + console.log(require('command-line-usage')([ + { + header: 'arrow2csv', + content: 'Print a CSV from an Arrow file' + }, + { + header: 'Synopsis', + content: [ + '$ arrow2csv {underline file.arrow} [{bold --schema} column_name ...]', + '$ arrow2csv [{bold --schema} column_name ...] [{bold --file} {underline file.arrow}]', + '$ arrow2csv {bold -s} column_1 {bold -s} column_2 [{bold -f} {underline file.arrow}]', + '$ arrow2csv [{bold --help}]' + ] + }, + { + header: 'Options', + optionList: cliOpts() + }, + { + header: 'Example', + content: [ + '$ arrow2csv --schema foo baz --sep " , " -f simple.arrow', + '> "row_id", "foo: Int32", "baz: Utf8"', + '> 0, 1, "aa"', + '> 1, null, null', + '> 2, 3, null', + '> 3, 4, "bbb"', + '> 4, 5, "cccc"', + ] + } + ])); + return 1; +} diff --git a/src/arrow/js/src/builder.ts b/src/arrow/js/src/builder.ts new file mode 100644 index 000000000..86db95306 --- /dev/null +++ b/src/arrow/js/src/builder.ts @@ -0,0 +1,527 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from './vector'; +import { BufferType } from './enum'; +import { Data, Buffers } from './data'; +import { createIsValidFunction } from './builder/valid'; +import { BuilderType as B, VectorType as V} from './interfaces'; +import { BufferBuilder, BitmapBufferBuilder, DataBufferBuilder, OffsetsBufferBuilder } from './builder/buffer'; +import { + DataType, strideForType, + Float, Int, Decimal, FixedSizeBinary, + Date_, Time, Timestamp, Interval, + Utf8, Binary, List, Map_ +} from './type'; + +/** + * A set of options required to create a `Builder` instance for a given `DataType`. + * @see {@link Builder} + */ +export interface BuilderOptions<T extends DataType = any, TNull = any> { + type: T; + nullValues?: TNull[] | ReadonlyArray<TNull> | null; + children?: { [key: string]: BuilderOptions } | BuilderOptions[]; +} + +/** + * A set of options to create an Iterable or AsyncIterable `Builder` transform function. + * @see {@link Builder.throughIterable} + * @see {@link Builder.throughAsyncIterable} + */ + +export interface IterableBuilderOptions<T extends DataType = any, TNull = any> extends BuilderOptions<T, TNull> { + highWaterMark?: number; + queueingStrategy?: 'bytes' | 'count'; + dictionaryHashFunction?: (value: any) => string | number; + valueToChildTypeId?: (builder: Builder<T, TNull>, value: any, offset: number) => number; +} + +/** + * An abstract base class for types that construct Arrow Vectors from arbitrary JavaScript values. + * + * A `Builder` is responsible for writing arbitrary JavaScript values + * to ArrayBuffers and/or child Builders according to the Arrow specification + * for each DataType, creating or resizing the underlying ArrayBuffers as necessary. + * + * The `Builder` for each Arrow `DataType` handles converting and appending + * values for a given `DataType`. The high-level {@link Builder.new `Builder.new()`} convenience + * method creates the specific `Builder` subclass for the supplied `DataType`. + * + * Once created, `Builder` instances support both appending values to the end + * of the `Builder`, and random-access writes to specific indices + * (`Builder.prototype.append(value)` is a convenience method for + * `builder.set(builder.length, value)`). Appending or setting values beyond the + * Builder's current length may cause the builder to grow its underlying buffers + * or child Builders (if applicable) to accommodate the new values. + * + * After enough values have been written to a `Builder`, `Builder.prototype.flush()` + * will commit the values to the underlying ArrayBuffers (or child Builders). The + * internal Builder state will be reset, and an instance of `Data<T>` is returned. + * Alternatively, `Builder.prototype.toVector()` will flush the `Builder` and return + * an instance of `Vector<T>` instead. + * + * When there are no more values to write, use `Builder.prototype.finish()` to + * finalize the `Builder`. This does not reset the internal state, so it is + * necessary to call `Builder.prototype.flush()` or `toVector()` one last time + * if there are still values queued to be flushed. + * + * Note: calling `Builder.prototype.finish()` is required when using a `DictionaryBuilder`, + * because this is when it flushes the values that have been enqueued in its internal + * dictionary's `Builder`, and creates the `dictionaryVector` for the `Dictionary` `DataType`. + * + * ```ts + * import { Builder, Utf8 } from 'apache-arrow'; + * + * const utf8Builder = Builder.new({ + * type: new Utf8(), + * nullValues: [null, 'n/a'] + * }); + * + * utf8Builder + * .append('hello') + * .append('n/a') + * .append('world') + * .append(null); + * + * const utf8Vector = utf8Builder.finish().toVector(); + * + * console.log(utf8Vector.toJSON()); + * // > ["hello", null, "world", null] + * ``` + * + * @typeparam T The `DataType` of this `Builder`. + * @typeparam TNull The type(s) of values which will be considered null-value sentinels. + */ +export abstract class Builder<T extends DataType = any, TNull = any> { + + /** + * Create a `Builder` instance based on the `type` property of the supplied `options` object. + * @param {BuilderOptions<T, TNull>} options An object with a required `DataType` instance + * and other optional parameters to be passed to the `Builder` subclass for the given `type`. + * + * @typeparam T The `DataType` of the `Builder` to create. + * @typeparam TNull The type(s) of values which will be considered null-value sentinels. + * @nocollapse + */ + // @ts-ignore + public static new<T extends DataType = any, TNull = any>(options: BuilderOptions<T, TNull>): B<T, TNull> {} + + /** @nocollapse */ + // @ts-ignore + public static throughNode<T extends DataType = any, TNull = any>(options: import('./io/node/builder').BuilderDuplexOptions<T, TNull>): import('stream').Duplex { + throw new Error(`"throughNode" not available in this environment`); + } + /** @nocollapse */ + // @ts-ignore + public static throughDOM<T extends DataType = any, TNull = any>(options: import('./io/whatwg/builder').BuilderTransformOptions<T, TNull>): import('./io/whatwg/builder').BuilderTransform<T, TNull> { + throw new Error(`"throughDOM" not available in this environment`); + } + + /** + * Transform a synchronous `Iterable` of arbitrary JavaScript values into a + * sequence of Arrow Vector<T> following the chunking semantics defined in + * the supplied `options` argument. + * + * This function returns a function that accepts an `Iterable` of values to + * transform. When called, this function returns an Iterator of `Vector<T>`. + * + * The resulting `Iterator<Vector<T>>` yields Vectors based on the + * `queueingStrategy` and `highWaterMark` specified in the `options` argument. + * + * * If `queueingStrategy` is `"count"` (or omitted), The `Iterator<Vector<T>>` + * will flush the underlying `Builder` (and yield a new `Vector<T>`) once the + * Builder's `length` reaches or exceeds the supplied `highWaterMark`. + * * If `queueingStrategy` is `"bytes"`, the `Iterator<Vector<T>>` will flush + * the underlying `Builder` (and yield a new `Vector<T>`) once its `byteLength` + * reaches or exceeds the supplied `highWaterMark`. + * + * @param {IterableBuilderOptions<T, TNull>} options An object of properties which determine the `Builder` to create and the chunking semantics to use. + * @returns A function which accepts a JavaScript `Iterable` of values to + * write, and returns an `Iterator` that yields Vectors according + * to the chunking semantics defined in the `options` argument. + * @nocollapse + */ + public static throughIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) { + return throughIterable(options); + } + + /** + * Transform an `AsyncIterable` of arbitrary JavaScript values into a + * sequence of Arrow Vector<T> following the chunking semantics defined in + * the supplied `options` argument. + * + * This function returns a function that accepts an `AsyncIterable` of values to + * transform. When called, this function returns an AsyncIterator of `Vector<T>`. + * + * The resulting `AsyncIterator<Vector<T>>` yields Vectors based on the + * `queueingStrategy` and `highWaterMark` specified in the `options` argument. + * + * * If `queueingStrategy` is `"count"` (or omitted), The `AsyncIterator<Vector<T>>` + * will flush the underlying `Builder` (and yield a new `Vector<T>`) once the + * Builder's `length` reaches or exceeds the supplied `highWaterMark`. + * * If `queueingStrategy` is `"bytes"`, the `AsyncIterator<Vector<T>>` will flush + * the underlying `Builder` (and yield a new `Vector<T>`) once its `byteLength` + * reaches or exceeds the supplied `highWaterMark`. + * + * @param {IterableBuilderOptions<T, TNull>} options An object of properties which determine the `Builder` to create and the chunking semantics to use. + * @returns A function which accepts a JavaScript `AsyncIterable` of values + * to write, and returns an `AsyncIterator` that yields Vectors + * according to the chunking semantics defined in the `options` + * argument. + * @nocollapse + */ + public static throughAsyncIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) { + return throughAsyncIterable(options); + } + + /** + * Construct a builder with the given Arrow DataType with optional null values, + * which will be interpreted as "null" when set or appended to the `Builder`. + * @param {{ type: T, nullValues?: any[] }} options A `BuilderOptions` object used to create this `Builder`. + */ + constructor({ 'type': type, 'nullValues': nulls }: BuilderOptions<T, TNull>) { + this.type = type; + this.children = []; + this.nullValues = nulls; + this.stride = strideForType(type); + this._nulls = new BitmapBufferBuilder(); + if (nulls && nulls.length > 0) { + this._isValid = createIsValidFunction(nulls); + } + } + + /** + * The Builder's `DataType` instance. + * @readonly + */ + public type: T; + /** + * The number of values written to the `Builder` that haven't been flushed yet. + * @readonly + */ + public length = 0; + /** + * A boolean indicating whether `Builder.prototype.finish()` has been called on this `Builder`. + * @readonly + */ + public finished = false; + /** + * The number of elements in the underlying values TypedArray that + * represent a single logical element, determined by this Builder's + * `DataType`. This is 1 for most types, but is larger when the `DataType` + * is `Int64`, `Uint64`, `Decimal`, `DateMillisecond`, certain variants of + * `Interval`, `Time`, or `Timestamp`, `FixedSizeBinary`, and `FixedSizeList`. + * @readonly + */ + public readonly stride: number; + public readonly children: Builder[]; + /** + * The list of null-value sentinels for this `Builder`. When one of these values + * is written to the `Builder` (either via `Builder.prototype.set()` or `Builder.prototype.append()`), + * a 1-bit is written to this Builder's underlying null BitmapBufferBuilder. + * @readonly + */ + public readonly nullValues?: TNull[] | ReadonlyArray<TNull> | null; + + /** + * Flush the `Builder` and return a `Vector<T>`. + * @returns {Vector<T>} A `Vector<T>` of the flushed values. + */ + public toVector() { return Vector.new(this.flush()); } + + public get ArrayType() { return this.type.ArrayType; } + public get nullCount() { return this._nulls.numInvalid; } + public get numChildren() { return this.children.length; } + + /** + * @returns The aggregate length (in bytes) of the values that have been written. + */ + public get byteLength(): number { + let size = 0; + this._offsets && (size += this._offsets.byteLength); + this._values && (size += this._values.byteLength); + this._nulls && (size += this._nulls.byteLength); + this._typeIds && (size += this._typeIds.byteLength); + return this.children.reduce((size, child) => size + child.byteLength, size); + } + + /** + * @returns The aggregate number of rows that have been reserved to write new values. + */ + public get reservedLength(): number { + return this._nulls.reservedLength; + } + + /** + * @returns The aggregate length (in bytes) that has been reserved to write new values. + */ + public get reservedByteLength(): number { + let size = 0; + this._offsets && (size += this._offsets.reservedByteLength); + this._values && (size += this._values.reservedByteLength); + this._nulls && (size += this._nulls.reservedByteLength); + this._typeIds && (size += this._typeIds.reservedByteLength); + return this.children.reduce((size, child) => size + child.reservedByteLength, size); + } + + protected _offsets!: DataBufferBuilder<Int32Array>; + public get valueOffsets() { return this._offsets ? this._offsets.buffer : null; } + + protected _values!: BufferBuilder<T['TArray'], any>; + public get values() { return this._values ? this._values.buffer : null; } + + protected _nulls: BitmapBufferBuilder; + public get nullBitmap() { return this._nulls ? this._nulls.buffer : null; } + + protected _typeIds!: DataBufferBuilder<Int8Array>; + public get typeIds() { return this._typeIds ? this._typeIds.buffer : null; } + + protected _isValid!: (value: T['TValue'] | TNull) => boolean; + protected _setValue!: (inst: Builder<T>, index: number, value: T['TValue']) => void; + + /** + * Appends a value (or null) to this `Builder`. + * This is equivalent to `builder.set(builder.length, value)`. + * @param {T['TValue'] | TNull } value The value to append. + */ + public append(value: T['TValue'] | TNull) { return this.set(this.length, value); } + + /** + * Validates whether a value is valid (true), or null (false) + * @param {T['TValue'] | TNull } value The value to compare against null the value representations + */ + public isValid(value: T['TValue'] | TNull): boolean { return this._isValid(value); } + + /** + * Write a value (or null-value sentinel) at the supplied index. + * If the value matches one of the null-value representations, a 1-bit is + * written to the null `BitmapBufferBuilder`. Otherwise, a 0 is written to + * the null `BitmapBufferBuilder`, and the value is passed to + * `Builder.prototype.setValue()`. + * @param {number} index The index of the value to write. + * @param {T['TValue'] | TNull } value The value to write at the supplied index. + * @returns {this} The updated `Builder` instance. + */ + public set(index: number, value: T['TValue'] | TNull) { + if (this.setValid(index, this.isValid(value))) { + this.setValue(index, value); + } + return this; + } + + /** + * Write a value to the underlying buffers at the supplied index, bypassing + * the null-value check. This is a low-level method that + * @param {number} index + * @param {T['TValue'] | TNull } value + */ + public setValue(index: number, value: T['TValue']) { this._setValue(this, index, value); } + public setValid(index: number, valid: boolean) { + this.length = this._nulls.set(index, +valid).length; + return valid; + } + + // @ts-ignore + public addChild(child: Builder, name = `${this.numChildren}`) { + throw new Error(`Cannot append children to non-nested type "${this.type}"`); + } + + /** + * Retrieve the child `Builder` at the supplied `index`, or null if no child + * exists at that index. + * @param {number} index The index of the child `Builder` to retrieve. + * @returns {Builder | null} The child Builder at the supplied index or null. + */ + public getChildAt<R extends DataType = any>(index: number): Builder<R> | null { + return this.children[index] || null; + } + + /** + * Commit all the values that have been written to their underlying + * ArrayBuffers, including any child Builders if applicable, and reset + * the internal `Builder` state. + * @returns A `Data<T>` of the buffers and childData representing the values written. + */ + public flush() { + + const buffers: any = []; + const values = this._values; + const offsets = this._offsets; + const typeIds = this._typeIds; + const { length, nullCount } = this; + + if (typeIds) { /* Unions */ + buffers[BufferType.TYPE] = typeIds.flush(length); + // DenseUnions + offsets && (buffers[BufferType.OFFSET] = offsets.flush(length)); + } else if (offsets) { /* Variable-width primitives (Binary, Utf8) and Lists */ + // Binary, Utf8 + values && (buffers[BufferType.DATA] = values.flush(offsets.last())); + buffers[BufferType.OFFSET] = offsets.flush(length); + } else if (values) { /* Fixed-width primitives (Int, Float, Decimal, Time, Timestamp, and Interval) */ + buffers[BufferType.DATA] = values.flush(length); + } + + nullCount > 0 && (buffers[BufferType.VALIDITY] = this._nulls.flush(length)); + + const data = Data.new<T>( + this.type, 0, length, nullCount, buffers as Buffers<T>, + this.children.map((child) => child.flush())) as Data<T>; + + this.clear(); + + return data; + } + + /** + * Finalize this `Builder`, and child builders if applicable. + * @returns {this} The finalized `Builder` instance. + */ + public finish() { + this.finished = true; + this.children.forEach((child) => child.finish()); + return this; + } + + /** + * Clear this Builder's internal state, including child Builders if applicable, and reset the length to 0. + * @returns {this} The cleared `Builder` instance. + */ + public clear() { + this.length = 0; + this._offsets && (this._offsets.clear()); + this._values && (this._values.clear()); + this._nulls && (this._nulls.clear()); + this._typeIds && (this._typeIds.clear()); + this.children.forEach((child) => child.clear()); + return this; + } +} + +(Builder.prototype as any).length = 1; +(Builder.prototype as any).stride = 1; +(Builder.prototype as any).children = null; +(Builder.prototype as any).finished = false; +(Builder.prototype as any).nullValues = null; +(Builder.prototype as any)._isValid = () => true; + +/** @ignore */ +export abstract class FixedWidthBuilder<T extends Int | Float | FixedSizeBinary | Date_ | Timestamp | Time | Decimal | Interval = any, TNull = any> extends Builder<T, TNull> { + constructor(opts: BuilderOptions<T, TNull>) { + super(opts); + this._values = new DataBufferBuilder(new this.ArrayType(0), this.stride); + } + public setValue(index: number, value: T['TValue']) { + const values = this._values; + values.reserve(index - values.length + 1); + return super.setValue(index, value); + } +} + +/** @ignore */ +export abstract class VariableWidthBuilder<T extends Binary | Utf8 | List | Map_, TNull = any> extends Builder<T, TNull> { + protected _pendingLength = 0; + protected _offsets: OffsetsBufferBuilder; + protected _pending: Map<number, any> | undefined; + constructor(opts: BuilderOptions<T, TNull>) { + super(opts); + this._offsets = new OffsetsBufferBuilder(); + } + public setValue(index: number, value: T['TValue']) { + const pending = this._pending || (this._pending = new Map()); + const current = pending.get(index); + current && (this._pendingLength -= current.length); + this._pendingLength += value.length; + pending.set(index, value); + } + public setValid(index: number, isValid: boolean) { + if (!super.setValid(index, isValid)) { + (this._pending || (this._pending = new Map())).set(index, undefined); + return false; + } + return true; + } + public clear() { + this._pendingLength = 0; + this._pending = undefined; + return super.clear(); + } + public flush() { + this._flush(); + return super.flush(); + } + public finish() { + this._flush(); + return super.finish(); + } + protected _flush() { + const pending = this._pending; + const pendingLength = this._pendingLength; + this._pendingLength = 0; + this._pending = undefined; + if (pending && pending.size > 0) { + this._flushPending(pending, pendingLength); + } + return this; + } + protected abstract _flushPending(pending: Map<number, any>, pendingLength: number): void; +} + +/** @ignore */ +type ThroughIterable<T extends DataType = any, TNull = any> = (source: Iterable<T['TValue'] | TNull>) => IterableIterator<V<T>>; + +/** @ignore */ +function throughIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) { + const { ['queueingStrategy']: queueingStrategy = 'count' } = options; + const { ['highWaterMark']: highWaterMark = queueingStrategy !== 'bytes' ? 1000 : 2 ** 14 } = options; + const sizeProperty: 'length' | 'byteLength' = queueingStrategy !== 'bytes' ? 'length' : 'byteLength'; + return function*(source: Iterable<T['TValue'] | TNull>) { + let numChunks = 0; + const builder = Builder.new(options); + for (const value of source) { + if (builder.append(value)[sizeProperty] >= highWaterMark) { + ++numChunks && (yield builder.toVector()); + } + } + if (builder.finish().length > 0 || numChunks === 0) { + yield builder.toVector(); + } + } as ThroughIterable<T, TNull>; +} + +/** @ignore */ +type ThroughAsyncIterable<T extends DataType = any, TNull = any> = (source: Iterable<T['TValue'] | TNull> | AsyncIterable<T['TValue'] | TNull>) => AsyncIterableIterator<V<T>>; + +/** @ignore */ +function throughAsyncIterable<T extends DataType = any, TNull = any>(options: IterableBuilderOptions<T, TNull>) { + const { ['queueingStrategy']: queueingStrategy = 'count' } = options; + const { ['highWaterMark']: highWaterMark = queueingStrategy !== 'bytes' ? 1000 : 2 ** 14 } = options; + const sizeProperty: 'length' | 'byteLength' = queueingStrategy !== 'bytes' ? 'length' : 'byteLength'; + return async function* (source: Iterable<T['TValue'] | TNull> | AsyncIterable<T['TValue'] | TNull>) { + let numChunks = 0; + const builder = Builder.new(options); + for await (const value of source) { + if (builder.append(value)[sizeProperty] >= highWaterMark) { + ++numChunks && (yield builder.toVector()); + } + } + if (builder.finish().length > 0 || numChunks === 0) { + yield builder.toVector(); + } + } as ThroughAsyncIterable<T, TNull>; +} diff --git a/src/arrow/js/src/builder/binary.ts b/src/arrow/js/src/builder/binary.ts new file mode 100644 index 000000000..829da5c97 --- /dev/null +++ b/src/arrow/js/src/builder/binary.ts @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Binary } from '../type'; +import { toUint8Array } from '../util/buffer'; +import { BufferBuilder } from './buffer'; +import { VariableWidthBuilder, BuilderOptions } from '../builder'; + +/** @ignore */ +export class BinaryBuilder<TNull = any> extends VariableWidthBuilder<Binary, TNull> { + constructor(opts: BuilderOptions<Binary, TNull>) { + super(opts); + this._values = new BufferBuilder(new Uint8Array(0)); + } + public get byteLength(): number { + let size = this._pendingLength + (this.length * 4); + this._offsets && (size += this._offsets.byteLength); + this._values && (size += this._values.byteLength); + this._nulls && (size += this._nulls.byteLength); + return size; + } + public setValue(index: number, value: Uint8Array) { + return super.setValue(index, toUint8Array(value)); + } + protected _flushPending(pending: Map<number, Uint8Array | undefined>, pendingLength: number) { + const offsets = this._offsets; + const data = this._values.reserve(pendingLength).buffer; + let index = 0, length = 0, offset = 0, value: Uint8Array | undefined; + for ([index, value] of pending) { + if (value === undefined) { + offsets.set(index, 0); + } else { + length = value.length; + data.set(value, offset); + offsets.set(index, length); + offset += length; + } + } + } +} diff --git a/src/arrow/js/src/builder/bool.ts b/src/arrow/js/src/builder/bool.ts new file mode 100644 index 000000000..5c0e0950e --- /dev/null +++ b/src/arrow/js/src/builder/bool.ts @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Bool } from '../type'; +import { BitmapBufferBuilder } from './buffer'; +import { Builder, BuilderOptions } from '../builder'; + +/** @ignore */ +export class BoolBuilder<TNull = any> extends Builder<Bool, TNull> { + constructor(options: BuilderOptions<Bool, TNull>) { + super(options); + this._values = new BitmapBufferBuilder(); + } + public setValue(index: number, value: boolean) { + this._values.set(index, +value); + } +} diff --git a/src/arrow/js/src/builder/buffer.ts b/src/arrow/js/src/builder/buffer.ts new file mode 100644 index 000000000..3c20cc001 --- /dev/null +++ b/src/arrow/js/src/builder/buffer.ts @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { memcpy } from '../util/buffer'; +import { BigIntAvailable, BigInt64Array, BigUint64Array } from '../util/compat'; +import { + TypedArray, TypedArrayConstructor, + BigIntArray, BigIntArrayConstructor +} from '../interfaces'; + +/** @ignore */ type DataValue<T> = T extends TypedArray ? number : T extends BigIntArray ? WideValue<T> : T; +/** @ignore */ type WideValue<T extends BigIntArray> = T extends BigIntArray ? bigint | Int32Array | Uint32Array : never; +/** @ignore */ type ArrayCtor<T extends TypedArray | BigIntArray> = + T extends TypedArray ? TypedArrayConstructor<T> : + T extends BigIntArray ? BigIntArrayConstructor<T> : + any; + +/** @ignore */ +const roundLengthUpToNearest64Bytes = (len: number, BPE: number) => ((((len * BPE) + 63) & ~63) || 64) / BPE; +/** @ignore */ +const sliceOrExtendArray = <T extends TypedArray | BigIntArray>(arr: T, len = 0) => ( + arr.length >= len ? arr.subarray(0, len) : memcpy(new (arr.constructor as any)(len), arr, 0) +) as T; + +/** @ignore */ +export interface BufferBuilder<T extends TypedArray | BigIntArray = any, TValue = DataValue<T>> { + readonly offset: number; +} + +/** @ignore */ +export class BufferBuilder<T extends TypedArray | BigIntArray = any, TValue = DataValue<T>> { + + constructor(buffer: T, stride = 1) { + this.buffer = buffer; + this.stride = stride; + this.BYTES_PER_ELEMENT = buffer.BYTES_PER_ELEMENT; + this.ArrayType = buffer.constructor as ArrayCtor<T>; + this._resize(this.length = buffer.length / stride | 0); + } + + public buffer: T; + public length: number; + public readonly stride: number; + public readonly ArrayType: ArrayCtor<T>; + public readonly BYTES_PER_ELEMENT: number; + + public get byteLength() { return this.length * this.stride * this.BYTES_PER_ELEMENT | 0; } + public get reservedLength() { return this.buffer.length / this.stride; } + public get reservedByteLength() { return this.buffer.byteLength; } + + // @ts-ignore + public set(index: number, value: TValue) { return this; } + public append(value: TValue) { return this.set(this.length, value); } + public reserve(extra: number) { + if (extra > 0) { + this.length += extra; + const stride = this.stride; + const length = this.length * stride; + const reserved = this.buffer.length; + if (length >= reserved) { + this._resize(reserved === 0 + ? roundLengthUpToNearest64Bytes(length * 1, this.BYTES_PER_ELEMENT) + : roundLengthUpToNearest64Bytes(length * 2, this.BYTES_PER_ELEMENT) + ); + } + } + return this; + } + public flush(length = this.length) { + length = roundLengthUpToNearest64Bytes(length * this.stride, this.BYTES_PER_ELEMENT); + const array = sliceOrExtendArray<T>(this.buffer, length); + this.clear(); + return array; + } + public clear() { + this.length = 0; + this._resize(0); + return this; + } + protected _resize(newLength: number) { + return this.buffer = <T> memcpy(new this.ArrayType(newLength), this.buffer); + } +} + +(BufferBuilder.prototype as any).offset = 0; + +/** @ignore */ +export class DataBufferBuilder<T extends TypedArray> extends BufferBuilder<T, number> { + public last() { return this.get(this.length - 1); } + public get(index: number) { return this.buffer[index]; } + public set(index: number, value: number) { + this.reserve(index - this.length + 1); + this.buffer[index * this.stride] = value; + return this; + } +} + +/** @ignore */ +export class BitmapBufferBuilder extends DataBufferBuilder<Uint8Array> { + + constructor(data = new Uint8Array(0)) { super(data, 1 / 8); } + + public numValid = 0; + public get numInvalid() { return this.length - this.numValid; } + public get(idx: number) { return this.buffer[idx >> 3] >> idx % 8 & 1; } + public set(idx: number, val: number) { + const { buffer } = this.reserve(idx - this.length + 1); + const byte = idx >> 3, bit = idx % 8, cur = buffer[byte] >> bit & 1; + // If `val` is truthy and the current bit is 0, flip it to 1 and increment `numValid`. + // If `val` is falsey and the current bit is 1, flip it to 0 and decrement `numValid`. + val ? cur === 0 && ((buffer[byte] |= (1 << bit)), ++this.numValid) + : cur === 1 && ((buffer[byte] &= ~(1 << bit)), --this.numValid); + return this; + } + public clear() { + this.numValid = 0; + return super.clear(); + } +} + +/** @ignore */ +export class OffsetsBufferBuilder extends DataBufferBuilder<Int32Array> { + constructor(data = new Int32Array(1)) { super(data, 1); } + public append(value: number) { + return this.set(this.length - 1, value); + } + public set(index: number, value: number) { + const offset = this.length - 1; + const buffer = this.reserve(index - offset + 1).buffer; + if (offset < index++) { + buffer.fill(buffer[offset], offset, index); + } + buffer[index] = buffer[index - 1] + value; + return this; + } + public flush(length = this.length - 1) { + if (length > this.length) { + this.set(length - 1, 0); + } + return super.flush(length + 1); + } +} + +/** @ignore */ +export class WideBufferBuilder<T extends TypedArray, R extends BigIntArray> extends BufferBuilder<T, DataValue<T>> { + public buffer64!: R; + protected _ArrayType64!: BigIntArrayConstructor<R>; + public get ArrayType64() { + return this._ArrayType64 || (this._ArrayType64 = <BigIntArrayConstructor<R>> (this.buffer instanceof Int32Array ? BigInt64Array : BigUint64Array)); + } + public set(index: number, value: DataValue<T>) { + this.reserve(index - this.length + 1); + switch (typeof value) { + case 'bigint': this.buffer64[index] = value; break; + case 'number': this.buffer[index * this.stride] = value; break; + default: this.buffer.set(value as TypedArray, index * this.stride); + } + return this; + } + protected _resize(newLength: number) { + const data = super._resize(newLength); + const length = data.byteLength / (this.BYTES_PER_ELEMENT * this.stride); + if (BigIntAvailable) { + this.buffer64 = new this.ArrayType64(data.buffer, data.byteOffset, length); + } + return data; + } +} diff --git a/src/arrow/js/src/builder/date.ts b/src/arrow/js/src/builder/date.ts new file mode 100644 index 000000000..e9748b58c --- /dev/null +++ b/src/arrow/js/src/builder/date.ts @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { FixedWidthBuilder } from '../builder'; +import { Date_, DateDay, DateMillisecond } from '../type'; + +/** @ignore */ +export class DateBuilder<T extends Date_ = Date_, TNull = any> extends FixedWidthBuilder<T, TNull> {} +/** @ignore */ +export class DateDayBuilder<TNull = any> extends DateBuilder<DateDay, TNull> {} +/** @ignore */ +export class DateMillisecondBuilder<TNull = any> extends DateBuilder<DateMillisecond, TNull> {} diff --git a/src/arrow/js/src/builder/decimal.ts b/src/arrow/js/src/builder/decimal.ts new file mode 100644 index 000000000..5814abd5b --- /dev/null +++ b/src/arrow/js/src/builder/decimal.ts @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Decimal } from '../type'; +import { FixedWidthBuilder } from '../builder'; + +/** @ignore */ +export class DecimalBuilder<TNull = any> extends FixedWidthBuilder<Decimal, TNull> {} diff --git a/src/arrow/js/src/builder/dictionary.ts b/src/arrow/js/src/builder/dictionary.ts new file mode 100644 index 000000000..6602825dd --- /dev/null +++ b/src/arrow/js/src/builder/dictionary.ts @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from '../vector'; +import { IntBuilder } from './int'; +import { Dictionary, DataType } from '../type'; +import { Builder, BuilderOptions } from '../builder'; + +type DictionaryHashFunction = (x: any) => string | number; + +export interface DictionaryBuilderOptions<T extends DataType = any, TNull = any> extends BuilderOptions<T, TNull> { + dictionaryHashFunction?: DictionaryHashFunction; +} + +/** @ignore */ +export class DictionaryBuilder<T extends Dictionary, TNull = any> extends Builder<T, TNull> { + + protected _dictionaryOffset: number; + protected _dictionary?: Vector<T['dictionary']>; + protected _keysToIndices: { [key: string]: number }; + public readonly indices: IntBuilder<T['indices']>; + public readonly dictionary: Builder<T['dictionary']>; + + constructor({ 'type': type, 'nullValues': nulls, 'dictionaryHashFunction': hashFn }: DictionaryBuilderOptions<T, TNull>) { + super({ type: new Dictionary(type.dictionary, type.indices, type.id, type.isOrdered) as T }); + this._nulls = <any> null; + this._dictionaryOffset = 0; + this._keysToIndices = Object.create(null); + this.indices = Builder.new({ 'type': this.type.indices, 'nullValues': nulls }) as IntBuilder<T['indices']>; + this.dictionary = Builder.new({ 'type': this.type.dictionary, 'nullValues': null }) as Builder<T['dictionary']>; + if (typeof hashFn === 'function') { + this.valueToKey = hashFn; + } + } + + public get values() { return this.indices.values; } + public get nullCount() { return this.indices.nullCount; } + public get nullBitmap() { return this.indices.nullBitmap; } + public get byteLength() { return this.indices.byteLength + this.dictionary.byteLength; } + public get reservedLength() { return this.indices.reservedLength + this.dictionary.reservedLength; } + public get reservedByteLength() { return this.indices.reservedByteLength + this.dictionary.reservedByteLength; } + public isValid(value: T['TValue'] | TNull) { return this.indices.isValid(value); } + public setValid(index: number, valid: boolean) { + const indices = this.indices; + valid = indices.setValid(index, valid); + this.length = indices.length; + return valid; + } + public setValue(index: number, value: T['TValue']) { + const keysToIndices = this._keysToIndices; + const key = this.valueToKey(value); + let idx = keysToIndices[key]; + if (idx === undefined) { + keysToIndices[key] = idx = this._dictionaryOffset + this.dictionary.append(value).length - 1; + } + return this.indices.setValue(index, idx); + } + public flush() { + const type = this.type; + const prev = this._dictionary; + const curr = this.dictionary.toVector(); + const data = this.indices.flush().clone(type); + data.dictionary = prev ? prev.concat(curr) : curr; + this.finished || (this._dictionaryOffset += curr.length); + this._dictionary = data.dictionary as Vector<T['dictionary']>; + this.clear(); + return data; + } + public finish() { + this.indices.finish(); + this.dictionary.finish(); + this._dictionaryOffset = 0; + this._keysToIndices = Object.create(null); + return super.finish(); + } + public clear() { + this.indices.clear(); + this.dictionary.clear(); + return super.clear(); + } + public valueToKey(val: any): string | number { + return typeof val === 'string' ? val : `${val}`; + } +} diff --git a/src/arrow/js/src/builder/fixedsizebinary.ts b/src/arrow/js/src/builder/fixedsizebinary.ts new file mode 100644 index 000000000..99aaf46a3 --- /dev/null +++ b/src/arrow/js/src/builder/fixedsizebinary.ts @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { FixedSizeBinary } from '../type'; +import { FixedWidthBuilder } from '../builder'; + +/** @ignore */ +export class FixedSizeBinaryBuilder<TNull = any> extends FixedWidthBuilder<FixedSizeBinary, TNull> {} diff --git a/src/arrow/js/src/builder/fixedsizelist.ts b/src/arrow/js/src/builder/fixedsizelist.ts new file mode 100644 index 000000000..cc20f5ba2 --- /dev/null +++ b/src/arrow/js/src/builder/fixedsizelist.ts @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Run } from './run'; +import { Field } from '../schema'; +import { Builder } from '../builder'; +import { DataType, FixedSizeList } from '../type'; + +/** @ignore */ +export class FixedSizeListBuilder<T extends DataType = any, TNull = any> extends Builder<FixedSizeList<T>, TNull> { + protected _run = new Run<T, TNull>(); + public setValue(index: number, value: T['TValue']) { + super.setValue(index, this._run.bind(value)); + } + public addChild(child: Builder<T>, name = '0') { + if (this.numChildren > 0) { + throw new Error('FixedSizeListBuilder can only have one child.'); + } + const childIndex = this.children.push(child); + this.type = new FixedSizeList(this.type.listSize, new Field(name, child.type, true)); + return childIndex; + } + public clear() { + this._run.clear(); + return super.clear(); + } +} diff --git a/src/arrow/js/src/builder/float.ts b/src/arrow/js/src/builder/float.ts new file mode 100644 index 000000000..dbf4c0d06 --- /dev/null +++ b/src/arrow/js/src/builder/float.ts @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { float64ToUint16 } from '../util/math'; +import { FixedWidthBuilder } from '../builder'; +import { Float, Float16, Float32, Float64 } from '../type'; + +/** @ignore */ +export class FloatBuilder<T extends Float = Float, TNull = any> extends FixedWidthBuilder<T, TNull> {} + +/** @ignore */ +export class Float16Builder<TNull = any> extends FloatBuilder<Float16, TNull> { + public setValue(index: number, value: number) { + // convert JS float64 to a uint16 + this._values.set(index, float64ToUint16(value)); + } +} + +/** @ignore */ +export class Float32Builder<TNull = any> extends FloatBuilder<Float32, TNull> { + public setValue(index: number, value: number) { + this._values.set(index, value); + } +} + +/** @ignore */ +export class Float64Builder<TNull = any> extends FloatBuilder<Float64, TNull> { + public setValue(index: number, value: number) { + this._values.set(index, value); + } +} diff --git a/src/arrow/js/src/builder/index.ts b/src/arrow/js/src/builder/index.ts new file mode 100644 index 000000000..dfd9d54f1 --- /dev/null +++ b/src/arrow/js/src/builder/index.ts @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/** @ignore */ +export { Builder, BuilderOptions } from '../builder'; +export { BoolBuilder } from './bool'; +export { NullBuilder } from './null'; +export { DateBuilder, DateDayBuilder, DateMillisecondBuilder } from './date'; +export { DecimalBuilder } from './decimal'; +export { DictionaryBuilder } from './dictionary'; +export { FixedSizeBinaryBuilder } from './fixedsizebinary'; +export { FloatBuilder, Float16Builder, Float32Builder, Float64Builder } from './float'; +export { IntBuilder, Int8Builder, Int16Builder, Int32Builder, Int64Builder, Uint8Builder, Uint16Builder, Uint32Builder, Uint64Builder } from './int'; +export { TimeBuilder, TimeSecondBuilder, TimeMillisecondBuilder, TimeMicrosecondBuilder, TimeNanosecondBuilder } from './time'; +export { TimestampBuilder, TimestampSecondBuilder, TimestampMillisecondBuilder, TimestampMicrosecondBuilder, TimestampNanosecondBuilder } from './timestamp'; +export { IntervalBuilder, IntervalDayTimeBuilder, IntervalYearMonthBuilder } from './interval'; +export { Utf8Builder } from './utf8'; +export { BinaryBuilder } from './binary'; +export { ListBuilder } from './list'; +export { FixedSizeListBuilder } from './fixedsizelist'; +export { MapBuilder } from './map'; +export { StructBuilder } from './struct'; +export { UnionBuilder, SparseUnionBuilder, DenseUnionBuilder } from './union'; + +import { Type } from '../enum'; +import { Field } from '../schema'; +import { DataType } from '../type'; +import { Utf8Builder } from './utf8'; +import { BuilderType as B } from '../interfaces'; +import { Builder, BuilderOptions } from '../builder'; +import { instance as setVisitor } from '../visitor/set'; +import { instance as getBuilderConstructor } from '../visitor/builderctor'; + +/** @nocollapse */ +Builder.new = newBuilder; + +function newBuilder<T extends DataType = any, TNull = any>(options: BuilderOptions<T, TNull>): B<T, TNull> { + + const type = options.type; + const builder = new (getBuilderConstructor.getVisitFn<T>(type)())(options) as Builder<T, TNull>; + + if (type.children && type.children.length > 0) { + + const children = options['children'] || [] as BuilderOptions[]; + const defaultOptions = { 'nullValues': options['nullValues'] }; + const getChildOptions = Array.isArray(children) + ? ((_: Field, i: number) => children[i] || defaultOptions) + : (({ name }: Field) => children[name] || defaultOptions); + + type.children.forEach((field, index) => { + const { type } = field; + const opts = getChildOptions(field, index); + builder.children.push(newBuilder({ ...opts, type })); + }); + } + + return builder as B<T, TNull>; +} + +(Object.keys(Type) as any[]) + .map((T: any) => Type[T] as any) + .filter((T: any): T is Type => typeof T === 'number' && T !== Type.NONE) + .forEach((typeId) => { + const BuilderCtor = getBuilderConstructor.visit(typeId); + BuilderCtor.prototype._setValue = setVisitor.getVisitFn(typeId); + }); + +(Utf8Builder.prototype as any)._setValue = setVisitor.visitBinary; diff --git a/src/arrow/js/src/builder/int.ts b/src/arrow/js/src/builder/int.ts new file mode 100644 index 000000000..5777bd125 --- /dev/null +++ b/src/arrow/js/src/builder/int.ts @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { bignumToBigInt } from '../util/bn'; +import { WideBufferBuilder } from './buffer'; +import { BigInt64Array } from '../util/compat'; +import { FixedWidthBuilder, BuilderOptions } from '../builder'; +import { Int, Int8, Int16, Int32, Int64, Uint8, Uint16, Uint32, Uint64 } from '../type'; + +/** @ignore */ +export class IntBuilder<T extends Int = Int, TNull = any> extends FixedWidthBuilder<T, TNull> { + public setValue(index: number, value: T['TValue']) { + this._values.set(index, value); + } +} + +/** @ignore */ +export class Int8Builder<TNull = any> extends IntBuilder<Int8, TNull> {} +/** @ignore */ +export class Int16Builder<TNull = any> extends IntBuilder<Int16, TNull> {} +/** @ignore */ +export class Int32Builder<TNull = any> extends IntBuilder<Int32, TNull> {} +/** @ignore */ +export class Int64Builder<TNull = any> extends IntBuilder<Int64, TNull> { + protected _values: WideBufferBuilder<Int32Array, BigInt64Array>; + constructor(options: BuilderOptions<Int64, TNull>) { + if (options['nullValues']) { + options['nullValues'] = (options['nullValues'] as TNull[]).map(toBigInt); + } + super(options); + this._values = new WideBufferBuilder(new Int32Array(0), 2); + } + public get values64() { return this._values.buffer64; } + public isValid(value: Int32Array | bigint | TNull) { return super.isValid(toBigInt(value)); } +} + +/** @ignore */ +export class Uint8Builder<TNull = any> extends IntBuilder<Uint8, TNull> {} +/** @ignore */ +export class Uint16Builder<TNull = any> extends IntBuilder<Uint16, TNull> {} +/** @ignore */ +export class Uint32Builder<TNull = any> extends IntBuilder<Uint32, TNull> {} +/** @ignore */ +export class Uint64Builder<TNull = any> extends IntBuilder<Uint64, TNull> { + protected _values: WideBufferBuilder<Uint32Array, BigUint64Array>; + constructor(options: BuilderOptions<Uint64, TNull>) { + if (options['nullValues']) { + options['nullValues'] = (options['nullValues'] as TNull[]).map(toBigInt); + } + super(options); + this._values = new WideBufferBuilder(new Uint32Array(0), 2); + } + public get values64() { return this._values.buffer64; } + public isValid(value: Uint32Array | bigint | TNull) { return super.isValid(toBigInt(value)); } +} + +const toBigInt = ((memo: any) => (value: any) => { + if (ArrayBuffer.isView(value)) { + memo.buffer = value.buffer; + memo.byteOffset = value.byteOffset; + memo.byteLength = value.byteLength; + value = bignumToBigInt(memo); + memo.buffer = null; + } + return value; +})({ 'BigIntArray': BigInt64Array }); diff --git a/src/arrow/js/src/builder/interval.ts b/src/arrow/js/src/builder/interval.ts new file mode 100644 index 000000000..374228215 --- /dev/null +++ b/src/arrow/js/src/builder/interval.ts @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { FixedWidthBuilder } from '../builder'; +import { Interval, IntervalDayTime, IntervalYearMonth } from '../type'; + +/** @ignore */ +export class IntervalBuilder<T extends Interval = Interval, TNull = any> extends FixedWidthBuilder<T, TNull> {} +/** @ignore */ +export class IntervalDayTimeBuilder<TNull = any> extends IntervalBuilder<IntervalDayTime, TNull> {} +/** @ignore */ +export class IntervalYearMonthBuilder<TNull = any> extends IntervalBuilder<IntervalYearMonth, TNull> {} diff --git a/src/arrow/js/src/builder/list.ts b/src/arrow/js/src/builder/list.ts new file mode 100644 index 000000000..844681eae --- /dev/null +++ b/src/arrow/js/src/builder/list.ts @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Run } from './run'; +import { Field } from '../schema'; +import { DataType, List } from '../type'; +import { OffsetsBufferBuilder } from './buffer'; +import { Builder, BuilderOptions, VariableWidthBuilder } from '../builder'; + +/** @ignore */ +export class ListBuilder<T extends DataType = any, TNull = any> extends VariableWidthBuilder<List<T>, TNull> { + protected _run = new Run<T, TNull>(); + protected _offsets: OffsetsBufferBuilder; + constructor(opts: BuilderOptions<List<T>, TNull>) { + super(opts); + this._offsets = new OffsetsBufferBuilder(); + } + public addChild(child: Builder<T>, name = '0') { + if (this.numChildren > 0) { + throw new Error('ListBuilder can only have one child.'); + } + this.children[this.numChildren] = child; + this.type = new List(new Field(name, child.type, true)); + return this.numChildren - 1; + } + public clear() { + this._run.clear(); + return super.clear(); + } + protected _flushPending(pending: Map<number, T['TValue'] | undefined>) { + const run = this._run; + const offsets = this._offsets; + const setValue = this._setValue; + let index = 0, value: Uint8Array | undefined; + for ([index, value] of pending) { + if (value === undefined) { + offsets.set(index, 0); + } else { + offsets.set(index, value.length); + setValue(this, index, run.bind(value)); + } + } + } +} diff --git a/src/arrow/js/src/builder/map.ts b/src/arrow/js/src/builder/map.ts new file mode 100644 index 000000000..25affef2c --- /dev/null +++ b/src/arrow/js/src/builder/map.ts @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Field } from '../schema'; +import { DataType, Map_, Struct } from '../type'; +import { Builder, VariableWidthBuilder } from '../builder'; + +/** @ignore */ type MapValue<K extends DataType = any, V extends DataType = any> = Map_<K, V>['TValue']; +/** @ignore */ type MapValues<K extends DataType = any, V extends DataType = any> = Map<number, MapValue<K, V> | undefined>; +/** @ignore */ type MapValueExt<K extends DataType = any, V extends DataType = any> = MapValue<K, V> | { [key: string]: V } | { [key: number]: V } ; + +/** @ignore */ +export class MapBuilder<K extends DataType = any, V extends DataType = any, TNull = any> extends VariableWidthBuilder<Map_<K, V>, TNull> { + + protected _pending: MapValues<K, V> | undefined; + public set(index: number, value: MapValueExt<K, V> | TNull) { + return super.set(index, value as MapValue<K, V> | TNull); + } + + public setValue(index: number, value: MapValueExt<K, V>) { + value = value instanceof Map ? value : new Map(Object.entries(value)); + const pending = this._pending || (this._pending = new Map() as MapValues<K, V>); + const current = pending.get(index); + current && (this._pendingLength -= current.size); + this._pendingLength += value.size; + pending.set(index, value); + } + + public addChild(child: Builder<Struct<{ key: K; value: V }>>, name = `${this.numChildren}`) { + if (this.numChildren > 0) { + throw new Error('ListBuilder can only have one child.'); + } + this.children[this.numChildren] = child; + this.type = new Map_<K, V>(new Field(name, child.type, true), this.type.keysSorted); + return this.numChildren - 1; + } + + protected _flushPending(pending: MapValues<K, V>) { + const offsets = this._offsets; + const setValue = this._setValue; + pending.forEach((value, index) => { + if (value === undefined) { + offsets.set(index, 0); + } else { + offsets.set(index, value.size); + setValue(this, index, value); + } + }); + } +} diff --git a/src/arrow/js/src/builder/null.ts b/src/arrow/js/src/builder/null.ts new file mode 100644 index 000000000..4be3f063b --- /dev/null +++ b/src/arrow/js/src/builder/null.ts @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Null } from '../type'; +import { Builder } from '../builder'; + +/** @ignore */ +export class NullBuilder<TNull = any> extends Builder<Null, TNull> { + // @ts-ignore + public setValue(index: number, value: null) {} + public setValid(index: number, valid: boolean) { + this.length = Math.max(index + 1, this.length); + return valid; + } +} diff --git a/src/arrow/js/src/builder/run.ts b/src/arrow/js/src/builder/run.ts new file mode 100644 index 000000000..5239f51f2 --- /dev/null +++ b/src/arrow/js/src/builder/run.ts @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from '../vector'; +import { DataType } from '../type'; + +/** @ignore */ +export class Run<T extends DataType = any, TNull = any> { + protected _values!: ArrayLike<T['TValue'] | TNull>; + public get length() { return this._values.length; } + public get(index: number) { return this._values[index]; } + public clear() { this._values = <any> null; return this; } + public bind(values: Vector<T> | ArrayLike<T['TValue'] | TNull>) { + if (values instanceof Vector) { + return values; + } + this._values = values; + return this as any; + } +} diff --git a/src/arrow/js/src/builder/struct.ts b/src/arrow/js/src/builder/struct.ts new file mode 100644 index 000000000..4d12336ce --- /dev/null +++ b/src/arrow/js/src/builder/struct.ts @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Field } from '../schema'; +import { Builder } from '../builder'; +import { DataType, Struct } from '../type'; + +/** @ignore */ +export class StructBuilder<T extends { [key: string]: DataType } = any, TNull = any> extends Builder<Struct<T>, TNull> { + public addChild(child: Builder, name = `${this.numChildren}`) { + const childIndex = this.children.push(child); + this.type = new Struct([...this.type.children, new Field(name, child.type, true)]); + return childIndex; + } +} diff --git a/src/arrow/js/src/builder/time.ts b/src/arrow/js/src/builder/time.ts new file mode 100644 index 000000000..bfa71d2b5 --- /dev/null +++ b/src/arrow/js/src/builder/time.ts @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { FixedWidthBuilder } from '../builder'; +import { Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond } from '../type'; + +/** @ignore */ +export class TimeBuilder<T extends Time = Time, TNull = any> extends FixedWidthBuilder<T, TNull> {} +/** @ignore */ +export class TimeSecondBuilder<TNull = any> extends TimeBuilder<TimeSecond, TNull> {} +/** @ignore */ +export class TimeMillisecondBuilder<TNull = any> extends TimeBuilder<TimeMillisecond, TNull> {} +/** @ignore */ +export class TimeMicrosecondBuilder<TNull = any> extends TimeBuilder<TimeMicrosecond, TNull> {} +/** @ignore */ +export class TimeNanosecondBuilder<TNull = any> extends TimeBuilder<TimeNanosecond, TNull> {} diff --git a/src/arrow/js/src/builder/timestamp.ts b/src/arrow/js/src/builder/timestamp.ts new file mode 100644 index 000000000..49741e9ba --- /dev/null +++ b/src/arrow/js/src/builder/timestamp.ts @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { FixedWidthBuilder } from '../builder'; +import { Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond } from '../type'; + +/** @ignore */ +export class TimestampBuilder<T extends Timestamp = Timestamp, TNull = any> extends FixedWidthBuilder<T, TNull> {} +/** @ignore */ +export class TimestampSecondBuilder<TNull = any> extends TimestampBuilder<TimestampSecond, TNull> {} +/** @ignore */ +export class TimestampMillisecondBuilder<TNull = any> extends TimestampBuilder<TimestampMillisecond, TNull> {} +/** @ignore */ +export class TimestampMicrosecondBuilder<TNull = any> extends TimestampBuilder<TimestampMicrosecond, TNull> {} +/** @ignore */ +export class TimestampNanosecondBuilder<TNull = any> extends TimestampBuilder<TimestampNanosecond, TNull> {} diff --git a/src/arrow/js/src/builder/union.ts b/src/arrow/js/src/builder/union.ts new file mode 100644 index 000000000..18ac05bf6 --- /dev/null +++ b/src/arrow/js/src/builder/union.ts @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Field } from '../schema'; +import { DataBufferBuilder } from './buffer'; +import { Builder, BuilderOptions } from '../builder'; +import { Union, SparseUnion, DenseUnion } from '../type'; + +export interface UnionBuilderOptions<T extends Union = any, TNull = any> extends BuilderOptions<T, TNull> { + valueToChildTypeId?: (builder: UnionBuilder<T, TNull>, value: any, offset: number) => number; +} + +/** @ignore */ +export abstract class UnionBuilder<T extends Union, TNull = any> extends Builder<T, TNull> { + + protected _typeIds: DataBufferBuilder<Int8Array>; + + constructor(options: UnionBuilderOptions<T, TNull>) { + super(options); + this._typeIds = new DataBufferBuilder(new Int8Array(0), 1); + if (typeof options['valueToChildTypeId'] === 'function') { + this._valueToChildTypeId = options['valueToChildTypeId']; + } + } + + public get typeIdToChildIndex() { return this.type.typeIdToChildIndex; } + + public append(value: T['TValue'] | TNull, childTypeId?: number) { + return this.set(this.length, value, childTypeId); + } + + public set(index: number, value: T['TValue'] | TNull, childTypeId?: number) { + if (childTypeId === undefined) { + childTypeId = this._valueToChildTypeId(this, value, index); + } + if (this.setValid(index, this.isValid(value))) { + this.setValue(index, value, childTypeId); + } + return this; + } + + public setValue(index: number, value: T['TValue'], childTypeId?: number) { + this._typeIds.set(index, childTypeId!); + super.setValue(index, value); + } + + public addChild(child: Builder, name = `${this.children.length}`) { + const childTypeId = this.children.push(child); + const { type: { children, mode, typeIds } } = this; + const fields = [...children, new Field(name, child.type)]; + this.type = <T> new Union(mode, [...typeIds, childTypeId], fields); + return childTypeId; + } + + /** @ignore */ + // @ts-ignore + protected _valueToChildTypeId(builder: UnionBuilder<T, TNull>, value: any, offset: number): number { + throw new Error(`Cannot map UnionBuilder value to child typeId. \ +Pass the \`childTypeId\` as the second argument to unionBuilder.append(), \ +or supply a \`valueToChildTypeId\` function as part of the UnionBuilder constructor options.`); + } +} + +/** @ignore */ +export class SparseUnionBuilder<T extends SparseUnion, TNull = any> extends UnionBuilder<T, TNull> {} +/** @ignore */ +export class DenseUnionBuilder<T extends DenseUnion, TNull = any> extends UnionBuilder<T, TNull> { + + protected _offsets: DataBufferBuilder<Int32Array>; + + constructor(options: UnionBuilderOptions<T, TNull>) { + super(options); + this._offsets = new DataBufferBuilder(new Int32Array(0)); + } + + /** @ignore */ + public setValue(index: number, value: T['TValue'], childTypeId?: number) { + const childIndex = this.type.typeIdToChildIndex[childTypeId!]; + this._offsets.set(index, this.getChildAt(childIndex)!.length); + return super.setValue(index, value, childTypeId); + } +} diff --git a/src/arrow/js/src/builder/utf8.ts b/src/arrow/js/src/builder/utf8.ts new file mode 100644 index 000000000..7564cdad6 --- /dev/null +++ b/src/arrow/js/src/builder/utf8.ts @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Utf8 } from '../type'; +import { encodeUtf8 } from '../util/utf8'; +import { BinaryBuilder } from './binary'; +import { BufferBuilder } from './buffer'; +import { VariableWidthBuilder, BuilderOptions } from '../builder'; + +/** @ignore */ +export class Utf8Builder<TNull = any> extends VariableWidthBuilder<Utf8, TNull> { + constructor(opts: BuilderOptions<Utf8, TNull>) { + super(opts); + this._values = new BufferBuilder(new Uint8Array(0)); + } + public get byteLength(): number { + let size = this._pendingLength + (this.length * 4); + this._offsets && (size += this._offsets.byteLength); + this._values && (size += this._values.byteLength); + this._nulls && (size += this._nulls.byteLength); + return size; + } + public setValue(index: number, value: string) { + return super.setValue(index, encodeUtf8(value) as any); + } + // @ts-ignore + protected _flushPending(pending: Map<number, Uint8Array | undefined>, pendingLength: number): void {} +} + +(Utf8Builder.prototype as any)._flushPending = (BinaryBuilder.prototype as any)._flushPending; diff --git a/src/arrow/js/src/builder/valid.ts b/src/arrow/js/src/builder/valid.ts new file mode 100644 index 000000000..ae5b799fb --- /dev/null +++ b/src/arrow/js/src/builder/valid.ts @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { DataType } from '../type'; +import { valueToString } from '../util/pretty'; +import { BigIntAvailable } from '../util/compat'; + +/** + * Dynamically compile the null values into an `isValid()` function whose + * implementation is a switch statement. Microbenchmarks in v8 indicate + * this approach is 25% faster than using an ES6 Map. + * + * @example + * console.log(createIsValidFunction([null, 'N/A', NaN])); + * `function (x) { + * if (x !== x) return false; + * switch (x) { + * case null: + * case "N/A": + * return false; + * } + * return true; + * }` + * + * @ignore + * @param nullValues + */ +export function createIsValidFunction<T extends DataType = any, TNull = any>(nullValues?: ReadonlyArray<TNull>) { + + if (!nullValues || nullValues.length <= 0) { + // @ts-ignore + return function isValid(value: any) { return true; }; + } + + let fnBody = ''; + const noNaNs = nullValues.filter((x) => x === x); + + if (noNaNs.length > 0) { + fnBody = ` + switch (x) {${noNaNs.map((x) => ` + case ${valueToCase(x)}:`).join('')} + return false; + }`; + } + + // NaN doesn't equal anything including itself, so it doesn't work as a + // switch case. Instead we must explicitly check for NaN before the switch. + if (nullValues.length !== noNaNs.length) { + fnBody = `if (x !== x) return false;\n${fnBody}`; + } + + return new Function(`x`, `${fnBody}\nreturn true;`) as (value: T['TValue'] | TNull) => boolean; +} + +/** @ignore */ +function valueToCase(x: any) { + if (typeof x !== 'bigint') { + return valueToString(x); + } else if (BigIntAvailable) { + return `${valueToString(x)}n`; + } + return `"${valueToString(x)}"`; +} diff --git a/src/arrow/js/src/column.ts b/src/arrow/js/src/column.ts new file mode 100644 index 000000000..48b40e5a1 --- /dev/null +++ b/src/arrow/js/src/column.ts @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from './data'; +import { Field } from './schema'; +import { DataType } from './type'; +import { Vector } from './vector'; +import { Clonable, Sliceable, Applicative } from './vector'; +import { VectorCtorArgs, VectorType as V } from './interfaces'; +import { Chunked, SearchContinuation } from './vector/chunked'; + +export interface Column<T extends DataType = any> { + concat(...others: Vector<T>[]): Column<T>; + slice(begin?: number, end?: number): Column<T>; + clone(chunks?: Vector<T>[], offsets?: Uint32Array): Column<T>; +} + +export class Column<T extends DataType = any> + extends Chunked<T> + implements Clonable<Column<T>>, + Sliceable<Column<T>>, + Applicative<T, Column<T>> { + + public static new<T extends DataType>(data: Data<T>, ...args: VectorCtorArgs<V<T>>): Column<T>; + public static new<T extends DataType>(field: string | Field<T>, ...chunks: (Vector<T> | Vector<T>[])[]): Column<T>; + public static new<T extends DataType>(field: string | Field<T>, data: Data<T>, ...args: VectorCtorArgs<V<T>>): Column<T>; + /** @nocollapse */ + public static new<T extends DataType = any>(...args: any[]) { + + let [field, data, ...rest] = args as [ + string | Field<T>, + Data<T> | Vector<T> | (Data<T> | Vector<T>)[], + ...any[] + ]; + + if (typeof field !== 'string' && !(field instanceof Field)) { + data = <Data<T> | Vector<T> | (Data<T> | Vector<T>)[]> field; + field = ''; + } + + const chunks = Chunked.flatten<T>( + Array.isArray(data) ? [...data, ...rest] : + data instanceof Vector ? [data, ...rest] : + [Vector.new(data, ...rest)] + ); + + if (typeof field === 'string') { + const type = chunks[0].data.type; + field = new Field(field, type, true); + } else if (!field.nullable && chunks.some(({ nullCount }) => nullCount > 0)) { + field = field.clone({ nullable: true }); + } + return new Column(field, chunks); + } + + constructor(field: Field<T>, vectors: Vector<T>[] = [], offsets?: Uint32Array) { + vectors = Chunked.flatten<T>(...vectors); + super(field.type, vectors, offsets); + this._field = field; + if (vectors.length === 1 && !(this instanceof SingleChunkColumn)) { + return new SingleChunkColumn(field, vectors[0], this._chunkOffsets); + } + } + + protected _field: Field<T>; + protected _children?: Column[]; + + public get field() { return this._field; } + public get name() { return this._field.name; } + public get nullable() { return this._field.nullable; } + public get metadata() { return this._field.metadata; } + + public clone(chunks = this._chunks) { + return new Column(this._field, chunks); + } + + public getChildAt<R extends DataType = any>(index: number): Column<R> | null { + + if (index < 0 || index >= this.numChildren) { return null; } + + const columns = this._children || (this._children = []); + let column: Column<R>, field: Field<R>, chunks: Vector<R>[]; + + if (column = columns[index]) { return column; } + if (field = ((this.type.children || [])[index] as Field<R>)) { + chunks = this._chunks + .map((vector) => vector.getChildAt<R>(index)) + .filter((vec): vec is Vector<R> => vec != null); + if (chunks.length > 0) { + return (columns[index] = new Column<R>(field, chunks)); + } + } + + return null; + } +} + +/** @ignore */ +class SingleChunkColumn<T extends DataType = any> extends Column<T> { + protected _chunk: Vector<T>; + constructor(field: Field<T>, vector: Vector<T>, offsets?: Uint32Array) { + super(field, [vector], offsets); + this._chunk = vector; + } + public search(index: number): [number, number] | null; + public search<N extends SearchContinuation<Chunked<T>>>(index: number, then?: N): ReturnType<N>; + public search<N extends SearchContinuation<Chunked<T>>>(index: number, then?: N) { + return then ? then(this, 0, index) : [0, index]; + } + public isValid(index: number): boolean { + return this._chunk.isValid(index); + } + public get(index: number): T['TValue'] | null { + return this._chunk.get(index); + } + public set(index: number, value: T['TValue'] | null): void { + this._chunk.set(index, value); + } + public indexOf(element: T['TValue'], offset?: number): number { + return this._chunk.indexOf(element, offset); + } +} diff --git a/src/arrow/js/src/compute/dataframe.ts b/src/arrow/js/src/compute/dataframe.ts new file mode 100644 index 000000000..e9df37194 --- /dev/null +++ b/src/arrow/js/src/compute/dataframe.ts @@ -0,0 +1,288 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Table } from '../table'; +import { Vector } from '../vector'; +import { IntVector } from '../vector/int'; +import { Field, Schema } from '../schema'; +import { Predicate, Col, PredicateFunc } from './predicate'; +import { RecordBatch } from '../recordbatch'; +import { VectorType as V } from '../interfaces'; +import { DataType, Int, Struct, Dictionary } from '../type'; + +/** @ignore */ +export type BindFunc = (batch: RecordBatch) => void; +/** @ignore */ +export type NextFunc = (idx: number, batch: RecordBatch) => void; + +/** + * `DataFrame` extends {@link Table} with support for predicate filtering. + * + * You can construct `DataFrames` like tables or convert a `Table` to a `DataFrame` + * with the constructor. + * + * ```ts + * const df = new DataFrame(table); + * ``` + */ +export class DataFrame<T extends { [key: string]: DataType } = any> extends Table<T> { + public filter(predicate: Predicate): FilteredDataFrame<T> { + return new FilteredDataFrame<T>(this.chunks, predicate); + } + public scan(next: NextFunc, bind?: BindFunc) { + const batches = this.chunks, numBatches = batches.length; + for (let batchIndex = -1; ++batchIndex < numBatches;) { + // load batches + const batch = batches[batchIndex]; + if (bind) { bind(batch); } + // yield all indices + for (let index = -1, numRows = batch.length; ++index < numRows;) { + next(index, batch); + } + } + } + public scanReverse(next: NextFunc, bind?: BindFunc) { + const batches = this.chunks, numBatches = batches.length; + for (let batchIndex = numBatches; --batchIndex >= 0;) { + // load batches + const batch = batches[batchIndex]; + if (bind) { bind(batch); } + // yield all indices + for (let index = batch.length; --index >= 0;) { + next(index, batch); + } + } + } + public countBy(name: Col | string) { + const batches = this.chunks, numBatches = batches.length; + const count_by = typeof name === 'string' ? new Col(name) : name as Col; + // Assume that all dictionary batches are deltas, which means that the + // last record batch has the most complete dictionary + count_by.bind(batches[numBatches - 1]); + const vector = count_by.vector as V<Dictionary>; + if (!DataType.isDictionary(vector.type)) { + throw new Error('countBy currently only supports dictionary-encoded columns'); + } + + const countByteLength = Math.ceil(Math.log(vector.length) / Math.log(256)); + const CountsArrayType = countByteLength == 4 ? Uint32Array : + countByteLength >= 2 ? Uint16Array : Uint8Array; + + const counts = new CountsArrayType(vector.dictionary.length); + for (let batchIndex = -1; ++batchIndex < numBatches;) { + // load batches + const batch = batches[batchIndex]; + // rebind the countBy Col + count_by.bind(batch); + const keys = (count_by.vector as V<Dictionary>).indices; + // yield all indices + for (let index = -1, numRows = batch.length; ++index < numRows;) { + const key = keys.get(index); + if (key !== null) { counts[key]++; } + } + } + return new CountByResult(vector.dictionary, IntVector.from(counts)); + } +} + +/** @ignore */ +export class CountByResult<T extends DataType = any, TCount extends Int = Int> extends Table<{ values: T; counts: TCount }> { + constructor(values: Vector<T>, counts: V<TCount>) { + type R = { values: T; counts: TCount }; + const schema = new Schema<R>([ + new Field('values', values.type), + new Field('counts', counts.type) + ]); + super(new RecordBatch<R>(schema, counts.length, [values, counts])); + } + public toJSON(): Record<string, unknown> { + const values = this.getColumnAt(0)!; + const counts = this.getColumnAt(1)!; + const result = {} as { [k: string]: number | null }; + for (let i = -1; ++i < this.length;) { + result[values.get(i)] = counts.get(i); + } + return result; + } +} + +/** @ignore */ +class FilteredBatchIterator<T extends { [key: string]: DataType }> implements IterableIterator<Struct<T>['TValue']> { + private batchIndex = 0; + private batch: RecordBatch<T>; + private index = 0; + private predicateFunc: PredicateFunc; + + constructor( + private batches: RecordBatch<T>[], + private predicate: Predicate + ) { + // TODO: bind batches lazily + // If predicate doesn't match anything in the batch we don't need + // to bind the callback + this.batch = this.batches[this.batchIndex]; + this.predicateFunc = this.predicate.bind(this.batch); + } + + next(): IteratorResult<Struct<T>['TValue']> { + while (this.batchIndex < this.batches.length) { + while (this.index < this.batch.length) { + if (this.predicateFunc(this.index, this.batch)) { + return { + value: this.batch.get(this.index++) as any, + }; + } + this.index++; + } + + if (++this.batchIndex < this.batches.length) { + this.index = 0; + this.batch = this.batches[this.batchIndex]; + this.predicateFunc = this.predicate.bind(this.batch); + } + } + + return {done: true, value: null}; + } + + [Symbol.iterator]() { + return this; + } +} + +/** @ignore */ +export class FilteredDataFrame<T extends { [key: string]: DataType } = any> extends DataFrame<T> { + private _predicate: Predicate; + constructor (batches: RecordBatch<T>[], predicate: Predicate) { + super(batches); + this._predicate = predicate; + } + public scan(next: NextFunc, bind?: BindFunc) { + // inlined version of this: + // this.parent.scan((idx, columns) => { + // if (this.predicate(idx, columns)) next(idx, columns); + // }); + const batches = this._chunks; + const numBatches = batches.length; + for (let batchIndex = -1; ++batchIndex < numBatches;) { + // load batches + const batch = batches[batchIndex]; + const predicate = this._predicate.bind(batch); + let isBound = false; + // yield all indices + for (let index = -1, numRows = batch.length; ++index < numRows;) { + if (predicate(index, batch)) { + // bind batches lazily - if predicate doesn't match anything + // in the batch we don't need to call bind on the batch + if (bind && !isBound) { + bind(batch); + isBound = true; + } + next(index, batch); + } + } + } + } + public scanReverse(next: NextFunc, bind?: BindFunc) { + const batches = this._chunks; + const numBatches = batches.length; + for (let batchIndex = numBatches; --batchIndex >= 0;) { + // load batches + const batch = batches[batchIndex]; + const predicate = this._predicate.bind(batch); + let isBound = false; + // yield all indices + for (let index = batch.length; --index >= 0;) { + if (predicate(index, batch)) { + // bind batches lazily - if predicate doesn't match anything + // in the batch we don't need to call bind on the batch + if (bind && !isBound) { + bind(batch); + isBound = true; + } + next(index, batch); + } + } + } + } + public count(): number { + // inlined version of this: + // let sum = 0; + // this.parent.scan((idx, columns) => { + // if (this.predicate(idx, columns)) ++sum; + // }); + // return sum; + let sum = 0; + const batches = this._chunks; + const numBatches = batches.length; + for (let batchIndex = -1; ++batchIndex < numBatches;) { + // load batches + const batch = batches[batchIndex]; + const predicate = this._predicate.bind(batch); + for (let index = -1, numRows = batch.length; ++index < numRows;) { + if (predicate(index, batch)) { ++sum; } + } + } + return sum; + } + + public [Symbol.iterator](): IterableIterator<Struct<T>['TValue']> { + // inlined version of this: + // this.parent.scan((idx, columns) => { + // if (this.predicate(idx, columns)) next(idx, columns); + // }); + return new FilteredBatchIterator<T>(this._chunks, this._predicate); + } + public filter(predicate: Predicate): FilteredDataFrame<T> { + return new FilteredDataFrame<T>( + this._chunks, + this._predicate.and(predicate) + ); + } + public countBy(name: Col | string) { + const batches = this._chunks, numBatches = batches.length; + const count_by = typeof name === 'string' ? new Col(name) : name as Col; + // Assume that all dictionary batches are deltas, which means that the + // last record batch has the most complete dictionary + count_by.bind(batches[numBatches - 1]); + const vector = count_by.vector as V<Dictionary>; + if (!DataType.isDictionary(vector.type)) { + throw new Error('countBy currently only supports dictionary-encoded columns'); + } + + const countByteLength = Math.ceil(Math.log(vector.length) / Math.log(256)); + const CountsArrayType = countByteLength == 4 ? Uint32Array : + countByteLength >= 2 ? Uint16Array : Uint8Array; + + const counts = new CountsArrayType(vector.dictionary.length); + + for (let batchIndex = -1; ++batchIndex < numBatches;) { + // load batches + const batch = batches[batchIndex]; + const predicate = this._predicate.bind(batch); + // rebind the countBy Col + count_by.bind(batch); + const keys = (count_by.vector as V<Dictionary>).indices; + // yield all indices + for (let index = -1, numRows = batch.length; ++index < numRows;) { + const key = keys.get(index); + if (key !== null && predicate(index, batch)) { counts[key]++; } + } + } + return new CountByResult(vector.dictionary, IntVector.from(counts)); + } +} diff --git a/src/arrow/js/src/compute/predicate.ts b/src/arrow/js/src/compute/predicate.ts new file mode 100644 index 000000000..52030763d --- /dev/null +++ b/src/arrow/js/src/compute/predicate.ts @@ -0,0 +1,292 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from '../vector'; +import { RecordBatch } from '../recordbatch'; +import { DictionaryVector } from '../vector/dictionary'; + +/** @ignore */ +export type ValueFunc<T> = (idx: number, cols: RecordBatch) => T | null; +/** @ignore */ +export type PredicateFunc = (idx: number, cols: RecordBatch) => boolean; + +/** @ignore */ +export abstract class Value<T> { + eq(other: Value<T> | T): Predicate { + if (!(other instanceof Value)) { other = new Literal(other); } + return new Equals(this, other); + } + le(other: Value<T> | T): Predicate { + if (!(other instanceof Value)) { other = new Literal(other); } + return new LTeq(this, other); + } + ge(other: Value<T> | T): Predicate { + if (!(other instanceof Value)) { other = new Literal(other); } + return new GTeq(this, other); + } + lt(other: Value<T> | T): Predicate { + return new Not(this.ge(other)); + } + gt(other: Value<T> | T): Predicate { + return new Not(this.le(other)); + } + ne(other: Value<T> | T): Predicate { + return new Not(this.eq(other)); + } +} + +/** @ignore */ +export class Literal<T= any> extends Value<T> { + constructor(public v: T) { super(); } +} + +/** @ignore */ +export class Col<T= any> extends Value<T> { + public vector!: Vector; + public colidx!: number; + + constructor(public name: string) { super(); } + bind(batch: RecordBatch): (idx: number, batch?: RecordBatch) => any { + if (!this.colidx) { + // Assume column index doesn't change between calls to bind + //this.colidx = cols.findIndex(v => v.name.indexOf(this.name) != -1); + this.colidx = -1; + const fields = batch.schema.fields; + for (let idx = -1; ++idx < fields.length;) { + if (fields[idx].name === this.name) { + this.colidx = idx; + break; + } + } + if (this.colidx < 0) { throw new Error(`Failed to bind Col "${this.name}"`); } + } + + const vec = this.vector = batch.getChildAt(this.colidx)!; + return (idx: number) => vec.get(idx); + } +} + +/** @ignore */ +export abstract class Predicate { + abstract bind(batch: RecordBatch): PredicateFunc; + and(...expr: Predicate[]): And { return new And(this, ...expr); } + or(...expr: Predicate[]): Or { return new Or(this, ...expr); } + not(): Predicate { return new Not(this); } +} + +/** @ignore */ +export abstract class ComparisonPredicate<T= any> extends Predicate { + constructor(public readonly left: Value<T>, public readonly right: Value<T>) { + super(); + } + + bind(batch: RecordBatch) { + if (this.left instanceof Literal) { + if (this.right instanceof Literal) { + return this._bindLitLit(batch, this.left, this.right); + } else { // right is a Col + + return this._bindLitCol(batch, this.left, this.right as Col); + } + } else { // left is a Col + if (this.right instanceof Literal) { + return this._bindColLit(batch, this.left as Col, this.right); + } else { // right is a Col + return this._bindColCol(batch, this.left as Col, this.right as Col); + } + } + } + + protected abstract _bindLitLit(batch: RecordBatch, left: Literal, right: Literal): PredicateFunc; + protected abstract _bindColCol(batch: RecordBatch, left: Col, right: Col): PredicateFunc; + protected abstract _bindColLit(batch: RecordBatch, col: Col, lit: Literal): PredicateFunc; + protected abstract _bindLitCol(batch: RecordBatch, lit: Literal, col: Col): PredicateFunc; +} + +/** @ignore */ +export abstract class CombinationPredicate extends Predicate { + readonly children: Predicate[]; + constructor(...children: Predicate[]) { + super(); + this.children = children; + } +} +// add children to prototype so it doesn't get mangled in es2015/umd +(<any> CombinationPredicate.prototype).children = Object.freeze([]); // freeze for safety + +/** @ignore */ +export class And extends CombinationPredicate { + constructor(...children: Predicate[]) { + // Flatten any Ands + children = children.reduce((accum: Predicate[], p: Predicate): Predicate[] => { + return accum.concat(p instanceof And ? p.children : p); + }, []); + super(...children); + } + bind(batch: RecordBatch) { + const bound = this.children.map((p) => p.bind(batch)); + return (idx: number, batch: RecordBatch) => bound.every((p) => p(idx, batch)); + } +} + +/** @ignore */ +export class Or extends CombinationPredicate { + constructor(...children: Predicate[]) { + // Flatten any Ors + children = children.reduce((accum: Predicate[], p: Predicate): Predicate[] => { + return accum.concat(p instanceof Or ? p.children : p); + }, []); + super(...children); + } + bind(batch: RecordBatch) { + const bound = this.children.map((p) => p.bind(batch)); + return (idx: number, batch: RecordBatch) => bound.some((p) => p(idx, batch)); + } +} + +/** @ignore */ +export class Equals extends ComparisonPredicate { + // Helpers used to cache dictionary reverse lookups between calls to bind + private lastDictionary: Vector|undefined; + private lastKey: number|undefined; + + protected _bindLitLit(_batch: RecordBatch, left: Literal, right: Literal): PredicateFunc { + const rtrn: boolean = left.v == right.v; + return () => rtrn; + } + + protected _bindColCol(batch: RecordBatch, left: Col, right: Col): PredicateFunc { + const left_func = left.bind(batch); + const right_func = right.bind(batch); + return (idx: number, batch: RecordBatch) => left_func(idx, batch) == right_func(idx, batch); + } + + protected _bindColLit(batch: RecordBatch, col: Col, lit: Literal): PredicateFunc { + const col_func = col.bind(batch); + if (col.vector instanceof DictionaryVector) { + let key: any; + const vector = col.vector as DictionaryVector; + if (vector.dictionary !== this.lastDictionary) { + key = vector.reverseLookup(lit.v); + this.lastDictionary = vector.dictionary; + this.lastKey = key; + } else { + key = this.lastKey; + } + + if (key === -1) { + // the value doesn't exist in the dictionary - always return + // false + // TODO: special-case of PredicateFunc that encapsulates this + // "always false" behavior. That way filtering operations don't + // have to bother checking + return () => false; + } else { + return (idx: number) => { + return vector.getKey(idx) === key; + }; + } + } else { + return (idx: number, cols: RecordBatch) => col_func(idx, cols) == lit.v; + } + } + + protected _bindLitCol(batch: RecordBatch, lit: Literal, col: Col) { + // Equals is commutative + return this._bindColLit(batch, col, lit); + } +} + +/** @ignore */ +export class LTeq extends ComparisonPredicate { + protected _bindLitLit(_batch: RecordBatch, left: Literal, right: Literal): PredicateFunc { + const rtrn: boolean = left.v <= right.v; + return () => rtrn; + } + + protected _bindColCol(batch: RecordBatch, left: Col, right: Col): PredicateFunc { + const left_func = left.bind(batch); + const right_func = right.bind(batch); + return (idx: number, cols: RecordBatch) => left_func(idx, cols) <= right_func(idx, cols); + } + + protected _bindColLit(batch: RecordBatch, col: Col, lit: Literal): PredicateFunc { + const col_func = col.bind(batch); + return (idx: number, cols: RecordBatch) => col_func(idx, cols) <= lit.v; + } + + protected _bindLitCol(batch: RecordBatch, lit: Literal, col: Col) { + const col_func = col.bind(batch); + return (idx: number, cols: RecordBatch) => lit.v <= col_func(idx, cols); + } +} + +/** @ignore */ +export class GTeq extends ComparisonPredicate { + protected _bindLitLit(_batch: RecordBatch, left: Literal, right: Literal): PredicateFunc { + const rtrn: boolean = left.v >= right.v; + return () => rtrn; + } + + protected _bindColCol(batch: RecordBatch, left: Col, right: Col): PredicateFunc { + const left_func = left.bind(batch); + const right_func = right.bind(batch); + return (idx: number, cols: RecordBatch) => left_func(idx, cols) >= right_func(idx, cols); + } + + protected _bindColLit(batch: RecordBatch, col: Col, lit: Literal): PredicateFunc { + const col_func = col.bind(batch); + return (idx: number, cols: RecordBatch) => col_func(idx, cols) >= lit.v; + } + + protected _bindLitCol(batch: RecordBatch, lit: Literal, col: Col) { + const col_func = col.bind(batch); + return (idx: number, cols: RecordBatch) => lit.v >= col_func(idx, cols); + } +} + +/** @ignore */ +export class Not extends Predicate { + constructor(public readonly child: Predicate) { + super(); + } + + bind(batch: RecordBatch) { + const func = this.child.bind(batch); + return (idx: number, batch: RecordBatch) => !func(idx, batch); + } +} + +/** @ignore */ +export class CustomPredicate extends Predicate { + constructor(private next: PredicateFunc, private bind_: (batch: RecordBatch) => void) { + super(); + } + + bind(batch: RecordBatch) { + this.bind_(batch); + return this.next; + } +} + +export function lit(v: any): Value<any> { return new Literal(v); } +export function col(n: string): Col<any> { return new Col(n); } +export function and(...p: Predicate[]): And { return new And(...p); } +export function or(...p: Predicate[]): Or { return new Or(...p); } +export function custom(next: PredicateFunc, bind: (batch: RecordBatch) => void) { + return new CustomPredicate(next, bind); +} diff --git a/src/arrow/js/src/data.ts b/src/arrow/js/src/data.ts new file mode 100644 index 000000000..2a549088c --- /dev/null +++ b/src/arrow/js/src/data.ts @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from './vector'; +import { truncateBitmap } from './util/bit'; +import { popcnt_bit_range } from './util/bit'; +import { BufferType, UnionMode, Type } from './enum'; +import { DataType, SparseUnion, DenseUnion, strideForType } from './type'; +import { toArrayBufferView, toUint8Array, toInt32Array } from './util/buffer'; +import { + Dictionary, + Null, Int, Float, + Binary, Bool, Utf8, Decimal, + Date_, Time, Timestamp, Interval, + List, Struct, Union, FixedSizeBinary, FixedSizeList, Map_, +} from './type'; + +// When slicing, we do not know the null count of the sliced range without +// doing some computation. To avoid doing this eagerly, we set the null count +// to -1 (any negative number will do). When Vector.nullCount is called the +// first time, the null count will be computed. See ARROW-33 +/** @ignore */ export type kUnknownNullCount = -1; +/** @ignore */ export const kUnknownNullCount = -1; + +/** @ignore */ export type NullBuffer = Uint8Array | null | undefined; +/** @ignore */ export type TypeIdsBuffer = Int8Array | ArrayLike<number> | Iterable<number> | undefined; +/** @ignore */ export type ValueOffsetsBuffer = Int32Array | ArrayLike<number> | Iterable<number> | undefined; +/** @ignore */ export type DataBuffer<T extends DataType> = T['TArray'] | ArrayLike<number> | Iterable<number> | undefined; + +/** @ignore */ +export interface Buffers<T extends DataType> { + [BufferType.OFFSET]: Int32Array; + [BufferType.DATA]: T['TArray']; + [BufferType.VALIDITY]: Uint8Array; + [BufferType.TYPE]: T['TArray']; +} + +/** @ignore */ +export interface Data<T extends DataType = DataType> { + readonly TType: T['TType']; + readonly TArray: T['TArray']; + readonly TValue: T['TValue']; +} + +/** @ignore */ +export class Data<T extends DataType = DataType> { + + public readonly type: T; + public readonly length: number; + public readonly offset: number; + public readonly stride: number; + public readonly childData: Data[]; + + /** + * The dictionary for this Vector, if any. Only used for Dictionary type. + */ + public dictionary?: Vector; + + public readonly values!: Buffers<T>[BufferType.DATA]; + public readonly typeIds!: Buffers<T>[BufferType.TYPE]; + public readonly nullBitmap!: Buffers<T>[BufferType.VALIDITY]; + public readonly valueOffsets!: Buffers<T>[BufferType.OFFSET]; + + public get typeId(): T['TType'] { return this.type.typeId; } + public get ArrayType(): T['ArrayType'] { return this.type.ArrayType; } + public get buffers() { + return [this.valueOffsets, this.values, this.nullBitmap, this.typeIds] as Buffers<T>; + } + public get byteLength(): number { + let byteLength = 0; + const { valueOffsets, values, nullBitmap, typeIds } = this; + valueOffsets && (byteLength += valueOffsets.byteLength); + values && (byteLength += values.byteLength); + nullBitmap && (byteLength += nullBitmap.byteLength); + typeIds && (byteLength += typeIds.byteLength); + return this.childData.reduce((byteLength, child) => byteLength + child.byteLength, byteLength); + } + + protected _nullCount: number | kUnknownNullCount; + + public get nullCount() { + let nullCount = this._nullCount; + let nullBitmap: Uint8Array | undefined; + if (nullCount <= kUnknownNullCount && (nullBitmap = this.nullBitmap)) { + this._nullCount = nullCount = this.length - popcnt_bit_range(nullBitmap, this.offset, this.offset + this.length); + } + return nullCount; + } + + constructor(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector) { + this.type = type; + this.dictionary = dictionary; + this.offset = Math.floor(Math.max(offset || 0, 0)); + this.length = Math.floor(Math.max(length || 0, 0)); + this._nullCount = Math.floor(Math.max(nullCount || 0, -1)); + this.childData = (childData || []).map((x) => x instanceof Data ? x : x.data) as Data[]; + let buffer: Buffers<T>[keyof Buffers<T>]; + if (buffers instanceof Data) { + this.stride = buffers.stride; + this.values = buffers.values; + this.typeIds = buffers.typeIds; + this.nullBitmap = buffers.nullBitmap; + this.valueOffsets = buffers.valueOffsets; + } else { + this.stride = strideForType(type); + if (buffers) { + (buffer = (buffers as Buffers<T>)[0]) && (this.valueOffsets = buffer); + (buffer = (buffers as Buffers<T>)[1]) && (this.values = buffer); + (buffer = (buffers as Buffers<T>)[2]) && (this.nullBitmap = buffer); + (buffer = (buffers as Buffers<T>)[3]) && (this.typeIds = buffer); + } + } + } + + public clone<R extends DataType>(type: R, offset = this.offset, length = this.length, nullCount = this._nullCount, buffers: Buffers<R> = <any> this, childData: (Data | Vector)[] = this.childData) { + return new Data(type, offset, length, nullCount, buffers, childData, this.dictionary); + } + + public slice(offset: number, length: number): Data<T> { + const { stride, typeId, childData } = this; + // +true === 1, +false === 0, so this means + // we keep nullCount at 0 if it's already 0, + // otherwise set to the invalidated flag -1 + const nullCount = +(this._nullCount === 0) - 1; + const childStride = typeId === 16 /* FixedSizeList */ ? stride : 1; + const buffers = this._sliceBuffers(offset, length, stride, typeId); + return this.clone<T>(this.type, this.offset + offset, length, nullCount, buffers, + // Don't slice children if we have value offsets (the variable-width types) + (!childData.length || this.valueOffsets) ? childData : this._sliceChildren(childData, childStride * offset, childStride * length)); + } + + public _changeLengthAndBackfillNullBitmap(newLength: number): Data<T> { + if (this.typeId === Type.Null) { + return this.clone(this.type, 0, newLength, 0); + } + const { length, nullCount } = this; + // start initialized with 0s (nulls), then fill from 0 to length with 1s (not null) + const bitmap = new Uint8Array(((newLength + 63) & ~63) >> 3).fill(255, 0, length >> 3); + // set all the bits in the last byte (up to bit `length - length % 8`) to 1 (not null) + bitmap[length >> 3] = (1 << (length - (length & ~7))) - 1; + // if we have a nullBitmap, truncate + slice and set it over the pre-filled 1s + if (nullCount > 0) { + bitmap.set(truncateBitmap(this.offset, length, this.nullBitmap), 0); + } + const buffers = this.buffers; + buffers[BufferType.VALIDITY] = bitmap; + return this.clone(this.type, 0, newLength, nullCount + (newLength - length), buffers); + } + + protected _sliceBuffers(offset: number, length: number, stride: number, typeId: T['TType']): Buffers<T> { + let arr: any; + const { buffers } = this; + // If typeIds exist, slice the typeIds buffer + (arr = buffers[BufferType.TYPE]) && (buffers[BufferType.TYPE] = arr.subarray(offset, offset + length)); + // If offsets exist, only slice the offsets buffer + (arr = buffers[BufferType.OFFSET]) && (buffers[BufferType.OFFSET] = arr.subarray(offset, offset + length + 1)) || + // Otherwise if no offsets, slice the data buffer. Don't slice the data vector for Booleans, since the offset goes by bits not bytes + (arr = buffers[BufferType.DATA]) && (buffers[BufferType.DATA] = typeId === 6 ? arr : arr.subarray(stride * offset, stride * (offset + length))); + return buffers; + } + + protected _sliceChildren(childData: Data[], offset: number, length: number): Data[] { + return childData.map((child) => child.slice(offset, length)); + } + + // + // Convenience methods for creating Data instances for each of the Arrow Vector types + // + /** @nocollapse */ + public static new<T extends DataType>(type: T, offset: number, length: number, nullCount?: number, buffers?: Partial<Buffers<T>> | Data<T>, childData?: (Data | Vector)[], dictionary?: Vector): Data<T> { + if (buffers instanceof Data) { buffers = buffers.buffers; } else if (!buffers) { buffers = [] as Partial<Buffers<T>>; } + switch (type.typeId) { + case Type.Null: return <unknown> Data.Null( <unknown> type as Null, offset, length) as Data<T>; + case Type.Int: return <unknown> Data.Int( <unknown> type as Int, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>; + case Type.Dictionary: return <unknown> Data.Dictionary( <unknown> type as Dictionary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || [], dictionary!) as Data<T>; + case Type.Float: return <unknown> Data.Float( <unknown> type as Float, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>; + case Type.Bool: return <unknown> Data.Bool( <unknown> type as Bool, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>; + case Type.Decimal: return <unknown> Data.Decimal( <unknown> type as Decimal, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>; + case Type.Date: return <unknown> Data.Date( <unknown> type as Date_, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>; + case Type.Time: return <unknown> Data.Time( <unknown> type as Time, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>; + case Type.Timestamp: return <unknown> Data.Timestamp( <unknown> type as Timestamp, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>; + case Type.Interval: return <unknown> Data.Interval( <unknown> type as Interval, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>; + case Type.FixedSizeBinary: return <unknown> Data.FixedSizeBinary( <unknown> type as FixedSizeBinary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.DATA] || []) as Data<T>; + case Type.Binary: return <unknown> Data.Binary( <unknown> type as Binary, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.OFFSET] || [], buffers[BufferType.DATA] || []) as Data<T>; + case Type.Utf8: return <unknown> Data.Utf8( <unknown> type as Utf8, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.OFFSET] || [], buffers[BufferType.DATA] || []) as Data<T>; + case Type.List: return <unknown> Data.List( <unknown> type as List, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.OFFSET] || [], (childData || [])[0]) as Data<T>; + case Type.FixedSizeList: return <unknown> Data.FixedSizeList( <unknown> type as FixedSizeList, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], (childData || [])[0]) as Data<T>; + case Type.Struct: return <unknown> Data.Struct( <unknown> type as Struct, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], childData || []) as Data<T>; + case Type.Map: return <unknown> Data.Map( <unknown> type as Map_, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.OFFSET] || [], (childData || [])[0]) as Data<T>; + case Type.Union: return <unknown> Data.Union( <unknown> type as Union, offset, length, nullCount || 0, buffers[BufferType.VALIDITY], buffers[BufferType.TYPE] || [], buffers[BufferType.OFFSET] || childData, childData) as Data<T>; + } + throw new Error(`Unrecognized typeId ${type.typeId}`); + } + + /** @nocollapse */ + public static Null<T extends Null>(type: T, offset: number, length: number) { + return new Data(type, offset, length, 0); + } + /** @nocollapse */ + public static Int<T extends Int>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) { + return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]); + } + /** @nocollapse */ + public static Dictionary<T extends Dictionary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>, dictionary: Vector<T['dictionary']>) { + return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView<T['TArray']>(type.indices.ArrayType, data), toUint8Array(nullBitmap)], [], dictionary); + } + /** @nocollapse */ + public static Float<T extends Float>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) { + return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]); + } + /** @nocollapse */ + public static Bool<T extends Bool>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) { + return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]); + } + /** @nocollapse */ + public static Decimal<T extends Decimal>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) { + return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]); + } + /** @nocollapse */ + public static Date<T extends Date_>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) { + return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]); + } + /** @nocollapse */ + public static Time<T extends Time>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) { + return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]); + } + /** @nocollapse */ + public static Timestamp<T extends Timestamp>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) { + return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]); + } + /** @nocollapse */ + public static Interval<T extends Interval>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) { + return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]); + } + /** @nocollapse */ + public static FixedSizeBinary<T extends FixedSizeBinary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, data: DataBuffer<T>) { + return new Data(type, offset, length, nullCount, [undefined, toArrayBufferView(type.ArrayType, data), toUint8Array(nullBitmap)]); + } + /** @nocollapse */ + public static Binary<T extends Binary>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, valueOffsets: ValueOffsetsBuffer, data: DataBuffer<T>) { + return new Data(type, offset, length, nullCount, [toInt32Array(valueOffsets), toUint8Array(data), toUint8Array(nullBitmap)]); + } + /** @nocollapse */ + public static Utf8<T extends Utf8>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, valueOffsets: ValueOffsetsBuffer, data: DataBuffer<T>) { + return new Data(type, offset, length, nullCount, [toInt32Array(valueOffsets), toUint8Array(data), toUint8Array(nullBitmap)]); + } + /** @nocollapse */ + public static List<T extends List>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, valueOffsets: ValueOffsetsBuffer, child: Data<T['valueType']> | Vector<T['valueType']>) { + return new Data(type, offset, length, nullCount, [toInt32Array(valueOffsets), undefined, toUint8Array(nullBitmap)], child ? [child] : []); + } + /** @nocollapse */ + public static FixedSizeList<T extends FixedSizeList>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, child: Data<T['valueType']> | Vector<T['valueType']>) { + return new Data(type, offset, length, nullCount, [undefined, undefined, toUint8Array(nullBitmap)], child ? [child] : []); + } + /** @nocollapse */ + public static Struct<T extends Struct>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, children: (Data | Vector)[]) { + return new Data(type, offset, length, nullCount, [undefined, undefined, toUint8Array(nullBitmap)], children); + } + /** @nocollapse */ + public static Map<T extends Map_>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, valueOffsets: ValueOffsetsBuffer, child: (Data | Vector)) { + return new Data(type, offset, length, nullCount, [toInt32Array(valueOffsets), undefined, toUint8Array(nullBitmap)], child ? [child] : []); + } + public static Union<T extends SparseUnion>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, typeIds: TypeIdsBuffer, children: (Data | Vector)[], _?: any): Data<T>; + public static Union<T extends DenseUnion>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, typeIds: TypeIdsBuffer, valueOffsets: ValueOffsetsBuffer, children: (Data | Vector)[]): Data<T>; + public static Union<T extends Union>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, typeIds: TypeIdsBuffer, valueOffsetsOrChildren: ValueOffsetsBuffer | (Data | Vector)[], children?: (Data | Vector)[]): Data<T>; + /** @nocollapse */ + public static Union<T extends Union>(type: T, offset: number, length: number, nullCount: number, nullBitmap: NullBuffer, typeIds: TypeIdsBuffer, valueOffsetsOrChildren: ValueOffsetsBuffer | (Data | Vector)[], children?: (Data | Vector)[]) { + const buffers = <unknown> [ + undefined, undefined, + toUint8Array(nullBitmap), + toArrayBufferView(type.ArrayType, typeIds) + ] as Partial<Buffers<T>>; + if (type.mode === UnionMode.Sparse) { + return new Data(type, offset, length, nullCount, buffers, valueOffsetsOrChildren as (Data | Vector)[]); + } + buffers[BufferType.OFFSET] = toInt32Array(<ValueOffsetsBuffer> valueOffsetsOrChildren); + return new Data(type, offset, length, nullCount, buffers, children); + } +} + +(Data.prototype as any).childData = Object.freeze([]); diff --git a/src/arrow/js/src/enum.ts b/src/arrow/js/src/enum.ts new file mode 100644 index 000000000..517aa27e8 --- /dev/null +++ b/src/arrow/js/src/enum.ts @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +export { + DateUnit, + TimeUnit, + Precision, + UnionMode, + IntervalUnit, + MetadataVersion, +} from './fb/Schema'; + +export { MessageHeader } from './fb/Message'; + +/** + * Main data type enumeration. + * + * Data types in this library are all *logical*. They can be expressed as + * either a primitive physical type (bytes or bits of some fixed size), a + * nested type consisting of other data types, or another data type (e.g. a + * timestamp encoded as an int64). + * + * **Note**: Only enum values 0-17 (NONE through Map) are written to an Arrow + * IPC payload. + * + * The rest of the values are specified here so TypeScript can narrow the type + * signatures further beyond the base Arrow Types. The Arrow DataTypes include + * metadata like `bitWidth` that impact the type signatures of the values we + * accept and return. + * + * For example, the `Int8Vector` reads 1-byte numbers from an `Int8Array`, an + * `Int32Vector` reads a 4-byte number from an `Int32Array`, and an `Int64Vector` + * reads a pair of 4-byte lo, hi 32-bit integers as a zero-copy slice from the + * underlying `Int32Array`. + * + * Library consumers benefit by knowing the narrowest type, since we can ensure + * the types across all public methods are propagated, and never bail to `any`. + * These values are _never_ used at runtime, and they will _never_ be written + * to the flatbuffers metadata of serialized Arrow IPC payloads. + */ +export enum Type { + /** The default placeholder type */ + NONE = 0, + /** A NULL type having no physical storage */ + Null = 1, + /** Signed or unsigned 8, 16, 32, or 64-bit little-endian integer */ + Int = 2, + /** 2, 4, or 8-byte floating point value */ + Float = 3, + /** Variable-length bytes (no guarantee of UTF8-ness) */ + Binary = 4, + /** UTF8 variable-length string as List<Char> */ + Utf8 = 5, + /** Boolean as 1 bit, LSB bit-packed ordering */ + Bool = 6, + /** Precision-and-scale-based decimal type. Storage type depends on the parameters. */ + Decimal = 7, + /** int32_t days or int64_t milliseconds since the UNIX epoch */ + Date = 8, + /** Time as signed 32 or 64-bit integer, representing either seconds, milliseconds, microseconds, or nanoseconds since midnight since midnight */ + Time = 9, + /** Exact timestamp encoded with int64 since UNIX epoch (Default unit millisecond) */ + Timestamp = 10, + /** YEAR_MONTH or DAY_TIME interval in SQL style */ + Interval = 11, + /** A list of some logical data type */ + List = 12, + /** Struct of logical types */ + Struct = 13, + /** Union of logical types */ + Union = 14, + /** Fixed-size binary. Each value occupies the same number of bytes */ + FixedSizeBinary = 15, + /** Fixed-size list. Each value occupies the same number of bytes */ + FixedSizeList = 16, + /** Map of named logical types */ + Map = 17, + + /** Dictionary aka Category type */ + Dictionary = -1, + Int8 = -2, + Int16 = -3, + Int32 = -4, + Int64 = -5, + Uint8 = -6, + Uint16 = -7, + Uint32 = -8, + Uint64 = -9, + Float16 = -10, + Float32 = -11, + Float64 = -12, + DateDay = -13, + DateMillisecond = -14, + TimestampSecond = -15, + TimestampMillisecond = -16, + TimestampMicrosecond = -17, + TimestampNanosecond = -18, + TimeSecond = -19, + TimeMillisecond = -20, + TimeMicrosecond = -21, + TimeNanosecond = -22, + DenseUnion = -23, + SparseUnion = -24, + IntervalDayTime = -25, + IntervalYearMonth = -26, +} + +export enum BufferType { + /** + * used in List type, Dense Union and variable length primitive types (String, Binary) + */ + OFFSET = 0, + + /** + * actual data, either wixed width primitive types in slots or variable width delimited by an OFFSET vector + */ + DATA = 1, + + /** + * Bit vector indicating if each value is null + */ + VALIDITY = 2, + + /** + * Type vector used in Union type + */ + TYPE = 3 + } diff --git a/src/arrow/js/src/fb/.eslintrc.js b/src/arrow/js/src/fb/.eslintrc.js new file mode 100644 index 000000000..d448540e4 --- /dev/null +++ b/src/arrow/js/src/fb/.eslintrc.js @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + rules: { + "@typescript-eslint/no-require-imports": "off", + "@typescript-eslint/no-inferrable-types": "off" + }, +};
\ No newline at end of file diff --git a/src/arrow/js/src/fb/File.ts b/src/arrow/js/src/fb/File.ts new file mode 100644 index 000000000..5746dd183 --- /dev/null +++ b/src/arrow/js/src/fb/File.ts @@ -0,0 +1,300 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +import { flatbuffers } from 'flatbuffers'; +import * as NS13596923344997147894 from './Schema'; +/** + * ---------------------------------------------------------------------- + * Arrow File metadata + * + * + * @constructor + */ +export class Footer { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Footer + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Footer { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Footer= obj + * @returns Footer + */ + static getRootAsFooter(bb: flatbuffers.ByteBuffer, obj?: Footer): Footer { + return (obj || new Footer()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Footer= obj + * @returns Footer + */ + static getSizePrefixedRootAsFooter(bb: flatbuffers.ByteBuffer, obj?: Footer): Footer { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Footer()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @returns MetadataVersion + */ + version(): NS13596923344997147894.MetadataVersion { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : NS13596923344997147894.MetadataVersion.V1; + } + + /** + * @param Schema= obj + * @returns Schema|null + */ + schema(obj?: NS13596923344997147894.Schema): NS13596923344997147894.Schema | null { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? (obj || new NS13596923344997147894.Schema()).__init(this.bb!.__indirect(this.bb_pos + offset), this.bb!) : null; + } + + /** + * @param number index + * @param Block= obj + * @returns Block + */ + dictionaries(index: number, obj?: Block): Block | null { + const offset = this.bb!.__offset(this.bb_pos, 8); + return offset ? (obj || new Block()).__init(this.bb!.__vector(this.bb_pos + offset) + index * 24, this.bb!) : null; + } + + /** + * @returns number + */ + dictionariesLength(): number { + const offset = this.bb!.__offset(this.bb_pos, 8); + return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; + } + + /** + * @param number index + * @param Block= obj + * @returns Block + */ + recordBatches(index: number, obj?: Block): Block | null { + const offset = this.bb!.__offset(this.bb_pos, 10); + return offset ? (obj || new Block()).__init(this.bb!.__vector(this.bb_pos + offset) + index * 24, this.bb!) : null; + } + + /** + * @returns number + */ + recordBatchesLength(): number { + const offset = this.bb!.__offset(this.bb_pos, 10); + return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; + } + + /** + * User-defined metadata + * + * @param number index + * @param KeyValue= obj + * @returns KeyValue + */ + customMetadata(index: number, obj?: NS13596923344997147894.KeyValue): NS13596923344997147894.KeyValue | null { + const offset = this.bb!.__offset(this.bb_pos, 12); + return offset ? (obj || new NS13596923344997147894.KeyValue()).__init(this.bb!.__indirect(this.bb!.__vector(this.bb_pos + offset) + index * 4), this.bb!) : null; + } + + /** + * @returns number + */ + customMetadataLength(): number { + const offset = this.bb!.__offset(this.bb_pos, 12); + return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; + } + + /** + * @param flatbuffers.Builder builder + */ + static startFooter(builder: flatbuffers.Builder) { + builder.startObject(5); + } + + /** + * @param flatbuffers.Builder builder + * @param MetadataVersion version + */ + static addVersion(builder: flatbuffers.Builder, version: NS13596923344997147894.MetadataVersion) { + builder.addFieldInt16(0, version, NS13596923344997147894.MetadataVersion.V1); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset schemaOffset + */ + static addSchema(builder: flatbuffers.Builder, schemaOffset: flatbuffers.Offset) { + builder.addFieldOffset(1, schemaOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset dictionariesOffset + */ + static addDictionaries(builder: flatbuffers.Builder, dictionariesOffset: flatbuffers.Offset) { + builder.addFieldOffset(2, dictionariesOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param number numElems + */ + static startDictionariesVector(builder: flatbuffers.Builder, numElems: number) { + builder.startVector(24, numElems, 8); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset recordBatchesOffset + */ + static addRecordBatches(builder: flatbuffers.Builder, recordBatchesOffset: flatbuffers.Offset) { + builder.addFieldOffset(3, recordBatchesOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param number numElems + */ + static startRecordBatchesVector(builder: flatbuffers.Builder, numElems: number) { + builder.startVector(24, numElems, 8); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset customMetadataOffset + */ + static addCustomMetadata(builder: flatbuffers.Builder, customMetadataOffset: flatbuffers.Offset) { + builder.addFieldOffset(4, customMetadataOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param Array.<flatbuffers.Offset> data + * @returns flatbuffers.Offset + */ + static createCustomMetadataVector(builder: flatbuffers.Builder, data: flatbuffers.Offset[]): flatbuffers.Offset { + builder.startVector(4, data.length, 4); + for (let i = data.length - 1; i >= 0; i--) { + builder.addOffset(data[i]); + } + return builder.endVector(); + } + + /** + * @param flatbuffers.Builder builder + * @param number numElems + */ + static startCustomMetadataVector(builder: flatbuffers.Builder, numElems: number) { + builder.startVector(4, numElems, 4); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endFooter(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset offset + */ + static finishFooterBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset) { + builder.finish(offset); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset offset + */ + static finishSizePrefixedFooterBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset) { + builder.finish(offset, undefined, true); + } + + static createFooter(builder: flatbuffers.Builder, version: NS13596923344997147894.MetadataVersion, schemaOffset: flatbuffers.Offset, dictionariesOffset: flatbuffers.Offset, recordBatchesOffset: flatbuffers.Offset, customMetadataOffset: flatbuffers.Offset): flatbuffers.Offset { + Footer.startFooter(builder); + Footer.addVersion(builder, version); + Footer.addSchema(builder, schemaOffset); + Footer.addDictionaries(builder, dictionariesOffset); + Footer.addRecordBatches(builder, recordBatchesOffset); + Footer.addCustomMetadata(builder, customMetadataOffset); + return Footer.endFooter(builder); + } +} +/** + * @constructor + */ +export class Block { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Block + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Block { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * Index to the start of the RecordBlock (note this is past the Message header) + * + * @returns flatbuffers.Long + */ + offset(): flatbuffers.Long { + return this.bb!.readInt64(this.bb_pos); + } + + /** + * Length of the metadata + * + * @returns number + */ + metaDataLength(): number { + return this.bb!.readInt32(this.bb_pos + 8); + } + + /** + * Length of the data (this is aligned so there can be a gap between this and + * the metadata). + * + * @returns flatbuffers.Long + */ + bodyLength(): flatbuffers.Long { + return this.bb!.readInt64(this.bb_pos + 16); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Long offset + * @param number metaDataLength + * @param flatbuffers.Long bodyLength + * @returns flatbuffers.Offset + */ + static createBlock(builder: flatbuffers.Builder, offset: flatbuffers.Long, metaDataLength: number, bodyLength: flatbuffers.Long): flatbuffers.Offset { + builder.prep(8, 24); + builder.writeInt64(bodyLength); + builder.pad(4); + builder.writeInt32(metaDataLength); + builder.writeInt64(offset); + return builder.offset(); + } + +} diff --git a/src/arrow/js/src/fb/Message.ts b/src/arrow/js/src/fb/Message.ts new file mode 100644 index 000000000..973eb0425 --- /dev/null +++ b/src/arrow/js/src/fb/Message.ts @@ -0,0 +1,709 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +import { flatbuffers } from 'flatbuffers'; +import * as NS13596923344997147894 from './Schema'; +/** + * @enum {number} + */ +export enum CompressionType { + LZ4_FRAME = 0, + ZSTD = 1 +} + +/** + * Provided for forward compatibility in case we need to support different + * strategies for compressing the IPC message body (like whole-body + * compression rather than buffer-level) in the future + * + * @enum {number} + */ +export enum BodyCompressionMethod { + /** + * Each constituent buffer is first compressed with the indicated + * compressor, and then written with the uncompressed length in the first 8 + * bytes as a 64-bit little-endian signed integer followed by the compressed + * buffer bytes (and then padding as required by the protocol). The + * uncompressed length may be set to -1 to indicate that the data that + * follows is not compressed, which can be useful for cases where + * compression does not yield appreciable savings. + */ + BUFFER = 0 +} + +/** + * ---------------------------------------------------------------------- + * The root Message type + * This union enables us to easily send different message types without + * redundant storage, and in the future we can easily add new message types. + * + * Arrow implementations do not need to implement all of the message types, + * which may include experimental metadata types. For maximum compatibility, + * it is best to send data using RecordBatch + * + * @enum {number} + */ +export enum MessageHeader { + NONE = 0, + Schema = 1, + DictionaryBatch = 2, + RecordBatch = 3, + Tensor = 4, + SparseTensor = 5 +} + +/** + * ---------------------------------------------------------------------- + * Data structures for describing a table row batch (a collection of + * equal-length Arrow arrays) + * Metadata about a field at some level of a nested type tree (but not + * its children). + * + * For example, a List<Int16> with values [[1, 2, 3], null, [4], [5, 6], null] + * would have {length: 5, null_count: 2} for its List node, and {length: 6, + * null_count: 0} for its Int16 node, as separate FieldNode structs + * + * @constructor + */ +export class FieldNode { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns FieldNode + */ + __init(i: number, bb: flatbuffers.ByteBuffer): FieldNode { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * The number of value slots in the Arrow array at this level of a nested + * tree + * + * @returns flatbuffers.Long + */ + length(): flatbuffers.Long { + return this.bb!.readInt64(this.bb_pos); + } + + /** + * The number of observed nulls. Fields with null_count == 0 may choose not + * to write their physical validity bitmap out as a materialized buffer, + * instead setting the length of the bitmap buffer to 0. + * + * @returns flatbuffers.Long + */ + nullCount(): flatbuffers.Long { + return this.bb!.readInt64(this.bb_pos + 8); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Long length + * @param flatbuffers.Long null_count + * @returns flatbuffers.Offset + */ + static createFieldNode(builder: flatbuffers.Builder, length: flatbuffers.Long, null_count: flatbuffers.Long): flatbuffers.Offset { + builder.prep(8, 16); + builder.writeInt64(null_count); + builder.writeInt64(length); + return builder.offset(); + } + +} +/** + * Optional compression for the memory buffers constituting IPC message + * bodies. Intended for use with RecordBatch but could be used for other + * message types + * + * @constructor + */ +export class BodyCompression { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns BodyCompression + */ + __init(i: number, bb: flatbuffers.ByteBuffer): BodyCompression { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param BodyCompression= obj + * @returns BodyCompression + */ + static getRootAsBodyCompression(bb: flatbuffers.ByteBuffer, obj?: BodyCompression): BodyCompression { + return (obj || new BodyCompression()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param BodyCompression= obj + * @returns BodyCompression + */ + static getSizePrefixedRootAsBodyCompression(bb: flatbuffers.ByteBuffer, obj?: BodyCompression): BodyCompression { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new BodyCompression()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * Compressor library + * + * @returns CompressionType + */ + codec(): CompressionType { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? /** */ (this.bb!.readInt8(this.bb_pos + offset)) : CompressionType.LZ4_FRAME; + } + + /** + * Indicates the way the record batch body was compressed + * + * @returns BodyCompressionMethod + */ + method(): BodyCompressionMethod { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? /** */ (this.bb!.readInt8(this.bb_pos + offset)) : BodyCompressionMethod.BUFFER; + } + + /** + * @param flatbuffers.Builder builder + */ + static startBodyCompression(builder: flatbuffers.Builder) { + builder.startObject(2); + } + + /** + * @param flatbuffers.Builder builder + * @param CompressionType codec + */ + static addCodec(builder: flatbuffers.Builder, codec: CompressionType) { + builder.addFieldInt8(0, codec, CompressionType.LZ4_FRAME); + } + + /** + * @param flatbuffers.Builder builder + * @param BodyCompressionMethod method + */ + static addMethod(builder: flatbuffers.Builder, method: BodyCompressionMethod) { + builder.addFieldInt8(1, method, BodyCompressionMethod.BUFFER); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endBodyCompression(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createBodyCompression(builder: flatbuffers.Builder, codec: CompressionType, method: BodyCompressionMethod): flatbuffers.Offset { + BodyCompression.startBodyCompression(builder); + BodyCompression.addCodec(builder, codec); + BodyCompression.addMethod(builder, method); + return BodyCompression.endBodyCompression(builder); + } +} +/** + * A data header describing the shared memory layout of a "record" or "row" + * batch. Some systems call this a "row batch" internally and others a "record + * batch". + * + * @constructor + */ +export class RecordBatch { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns RecordBatch + */ + __init(i: number, bb: flatbuffers.ByteBuffer): RecordBatch { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param RecordBatch= obj + * @returns RecordBatch + */ + static getRootAsRecordBatch(bb: flatbuffers.ByteBuffer, obj?: RecordBatch): RecordBatch { + return (obj || new RecordBatch()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param RecordBatch= obj + * @returns RecordBatch + */ + static getSizePrefixedRootAsRecordBatch(bb: flatbuffers.ByteBuffer, obj?: RecordBatch): RecordBatch { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new RecordBatch()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * number of records / rows. The arrays in the batch should all have this + * length + * + * @returns flatbuffers.Long + */ + length(): flatbuffers.Long { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? this.bb!.readInt64(this.bb_pos + offset) : this.bb!.createLong(0, 0); + } + + /** + * Nodes correspond to the pre-ordered flattened logical schema + * + * @param number index + * @param FieldNode= obj + * @returns FieldNode + */ + nodes(index: number, obj?: FieldNode): FieldNode | null { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? (obj || new FieldNode()).__init(this.bb!.__vector(this.bb_pos + offset) + index * 16, this.bb!) : null; + } + + /** + * @returns number + */ + nodesLength(): number { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; + } + + /** + * Buffers correspond to the pre-ordered flattened buffer tree + * + * The number of buffers appended to this list depends on the schema. For + * example, most primitive arrays will have 2 buffers, 1 for the validity + * bitmap and 1 for the values. For struct arrays, there will only be a + * single buffer for the validity (nulls) bitmap + * + * @param number index + * @param Buffer= obj + * @returns Buffer + */ + buffers(index: number, obj?: NS13596923344997147894.Buffer): NS13596923344997147894.Buffer | null { + const offset = this.bb!.__offset(this.bb_pos, 8); + return offset ? (obj || new NS13596923344997147894.Buffer()).__init(this.bb!.__vector(this.bb_pos + offset) + index * 16, this.bb!) : null; + } + + /** + * @returns number + */ + buffersLength(): number { + const offset = this.bb!.__offset(this.bb_pos, 8); + return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; + } + + /** + * Optional compression of the message body + * + * @param BodyCompression= obj + * @returns BodyCompression|null + */ + compression(obj?: BodyCompression): BodyCompression | null { + const offset = this.bb!.__offset(this.bb_pos, 10); + return offset ? (obj || new BodyCompression()).__init(this.bb!.__indirect(this.bb_pos + offset), this.bb!) : null; + } + + /** + * @param flatbuffers.Builder builder + */ + static startRecordBatch(builder: flatbuffers.Builder) { + builder.startObject(4); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Long length + */ + static addLength(builder: flatbuffers.Builder, length: flatbuffers.Long) { + builder.addFieldInt64(0, length, builder.createLong(0, 0)); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset nodesOffset + */ + static addNodes(builder: flatbuffers.Builder, nodesOffset: flatbuffers.Offset) { + builder.addFieldOffset(1, nodesOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param number numElems + */ + static startNodesVector(builder: flatbuffers.Builder, numElems: number) { + builder.startVector(16, numElems, 8); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset buffersOffset + */ + static addBuffers(builder: flatbuffers.Builder, buffersOffset: flatbuffers.Offset) { + builder.addFieldOffset(2, buffersOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param number numElems + */ + static startBuffersVector(builder: flatbuffers.Builder, numElems: number) { + builder.startVector(16, numElems, 8); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset compressionOffset + */ + static addCompression(builder: flatbuffers.Builder, compressionOffset: flatbuffers.Offset) { + builder.addFieldOffset(3, compressionOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endRecordBatch(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createRecordBatch(builder: flatbuffers.Builder, length: flatbuffers.Long, nodesOffset: flatbuffers.Offset, buffersOffset: flatbuffers.Offset, compressionOffset: flatbuffers.Offset): flatbuffers.Offset { + RecordBatch.startRecordBatch(builder); + RecordBatch.addLength(builder, length); + RecordBatch.addNodes(builder, nodesOffset); + RecordBatch.addBuffers(builder, buffersOffset); + RecordBatch.addCompression(builder, compressionOffset); + return RecordBatch.endRecordBatch(builder); + } +} +/** + * For sending dictionary encoding information. Any Field can be + * dictionary-encoded, but in this case none of its children may be + * dictionary-encoded. + * There is one vector / column per dictionary, but that vector / column + * may be spread across multiple dictionary batches by using the isDelta + * flag + * + * @constructor + */ +export class DictionaryBatch { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns DictionaryBatch + */ + __init(i: number, bb: flatbuffers.ByteBuffer): DictionaryBatch { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param DictionaryBatch= obj + * @returns DictionaryBatch + */ + static getRootAsDictionaryBatch(bb: flatbuffers.ByteBuffer, obj?: DictionaryBatch): DictionaryBatch { + return (obj || new DictionaryBatch()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param DictionaryBatch= obj + * @returns DictionaryBatch + */ + static getSizePrefixedRootAsDictionaryBatch(bb: flatbuffers.ByteBuffer, obj?: DictionaryBatch): DictionaryBatch { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new DictionaryBatch()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @returns flatbuffers.Long + */ + id(): flatbuffers.Long { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? this.bb!.readInt64(this.bb_pos + offset) : this.bb!.createLong(0, 0); + } + + /** + * @param RecordBatch= obj + * @returns RecordBatch|null + */ + data(obj?: RecordBatch): RecordBatch | null { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? (obj || new RecordBatch()).__init(this.bb!.__indirect(this.bb_pos + offset), this.bb!) : null; + } + + /** + * If isDelta is true the values in the dictionary are to be appended to a + * dictionary with the indicated id. If isDelta is false this dictionary + * should replace the existing dictionary. + * + * @returns boolean + */ + isDelta(): boolean { + const offset = this.bb!.__offset(this.bb_pos, 8); + return offset ? !!this.bb!.readInt8(this.bb_pos + offset) : false; + } + + /** + * @param flatbuffers.Builder builder + */ + static startDictionaryBatch(builder: flatbuffers.Builder) { + builder.startObject(3); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Long id + */ + static addId(builder: flatbuffers.Builder, id: flatbuffers.Long) { + builder.addFieldInt64(0, id, builder.createLong(0, 0)); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset dataOffset + */ + static addData(builder: flatbuffers.Builder, dataOffset: flatbuffers.Offset) { + builder.addFieldOffset(1, dataOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param boolean isDelta + */ + static addIsDelta(builder: flatbuffers.Builder, isDelta: boolean) { + builder.addFieldInt8(2, +isDelta, +false); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endDictionaryBatch(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createDictionaryBatch(builder: flatbuffers.Builder, id: flatbuffers.Long, dataOffset: flatbuffers.Offset, isDelta: boolean): flatbuffers.Offset { + DictionaryBatch.startDictionaryBatch(builder); + DictionaryBatch.addId(builder, id); + DictionaryBatch.addData(builder, dataOffset); + DictionaryBatch.addIsDelta(builder, isDelta); + return DictionaryBatch.endDictionaryBatch(builder); + } +} +/** + * @constructor + */ +export class Message { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Message + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Message { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Message= obj + * @returns Message + */ + static getRootAsMessage(bb: flatbuffers.ByteBuffer, obj?: Message): Message { + return (obj || new Message()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Message= obj + * @returns Message + */ + static getSizePrefixedRootAsMessage(bb: flatbuffers.ByteBuffer, obj?: Message): Message { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Message()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @returns MetadataVersion + */ + version(): NS13596923344997147894.MetadataVersion { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : NS13596923344997147894.MetadataVersion.V1; + } + + /** + * @returns MessageHeader + */ + headerType(): MessageHeader { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? /** */ (this.bb!.readUint8(this.bb_pos + offset)) : MessageHeader.NONE; + } + + /** + * @param flatbuffers.Table obj + * @returns ?flatbuffers.Table + */ + header<T extends flatbuffers.Table>(obj: T): T | null { + const offset = this.bb!.__offset(this.bb_pos, 8); + return offset ? this.bb!.__union(obj, this.bb_pos + offset) : null; + } + + /** + * @returns flatbuffers.Long + */ + bodyLength(): flatbuffers.Long { + const offset = this.bb!.__offset(this.bb_pos, 10); + return offset ? this.bb!.readInt64(this.bb_pos + offset) : this.bb!.createLong(0, 0); + } + + /** + * @param number index + * @param KeyValue= obj + * @returns KeyValue + */ + customMetadata(index: number, obj?: NS13596923344997147894.KeyValue): NS13596923344997147894.KeyValue | null { + const offset = this.bb!.__offset(this.bb_pos, 12); + return offset ? (obj || new NS13596923344997147894.KeyValue()).__init(this.bb!.__indirect(this.bb!.__vector(this.bb_pos + offset) + index * 4), this.bb!) : null; + } + + /** + * @returns number + */ + customMetadataLength(): number { + const offset = this.bb!.__offset(this.bb_pos, 12); + return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; + } + + /** + * @param flatbuffers.Builder builder + */ + static startMessage(builder: flatbuffers.Builder) { + builder.startObject(5); + } + + /** + * @param flatbuffers.Builder builder + * @param MetadataVersion version + */ + static addVersion(builder: flatbuffers.Builder, version: NS13596923344997147894.MetadataVersion) { + builder.addFieldInt16(0, version, NS13596923344997147894.MetadataVersion.V1); + } + + /** + * @param flatbuffers.Builder builder + * @param MessageHeader headerType + */ + static addHeaderType(builder: flatbuffers.Builder, headerType: MessageHeader) { + builder.addFieldInt8(1, headerType, MessageHeader.NONE); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset headerOffset + */ + static addHeader(builder: flatbuffers.Builder, headerOffset: flatbuffers.Offset) { + builder.addFieldOffset(2, headerOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Long bodyLength + */ + static addBodyLength(builder: flatbuffers.Builder, bodyLength: flatbuffers.Long) { + builder.addFieldInt64(3, bodyLength, builder.createLong(0, 0)); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset customMetadataOffset + */ + static addCustomMetadata(builder: flatbuffers.Builder, customMetadataOffset: flatbuffers.Offset) { + builder.addFieldOffset(4, customMetadataOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param Array.<flatbuffers.Offset> data + * @returns flatbuffers.Offset + */ + static createCustomMetadataVector(builder: flatbuffers.Builder, data: flatbuffers.Offset[]): flatbuffers.Offset { + builder.startVector(4, data.length, 4); + for (let i = data.length - 1; i >= 0; i--) { + builder.addOffset(data[i]); + } + return builder.endVector(); + } + + /** + * @param flatbuffers.Builder builder + * @param number numElems + */ + static startCustomMetadataVector(builder: flatbuffers.Builder, numElems: number) { + builder.startVector(4, numElems, 4); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endMessage(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset offset + */ + static finishMessageBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset) { + builder.finish(offset); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset offset + */ + static finishSizePrefixedMessageBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset) { + builder.finish(offset, undefined, true); + } + + static createMessage(builder: flatbuffers.Builder, version: NS13596923344997147894.MetadataVersion, headerType: MessageHeader, headerOffset: flatbuffers.Offset, bodyLength: flatbuffers.Long, customMetadataOffset: flatbuffers.Offset): flatbuffers.Offset { + Message.startMessage(builder); + Message.addVersion(builder, version); + Message.addHeaderType(builder, headerType); + Message.addHeader(builder, headerOffset); + Message.addBodyLength(builder, bodyLength); + Message.addCustomMetadata(builder, customMetadataOffset); + return Message.endMessage(builder); + } +} diff --git a/src/arrow/js/src/fb/Schema.ts b/src/arrow/js/src/fb/Schema.ts new file mode 100644 index 000000000..f675bc2a0 --- /dev/null +++ b/src/arrow/js/src/fb/Schema.ts @@ -0,0 +1,2658 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +import { flatbuffers } from 'flatbuffers'; +/** + * Logical types, vector layouts, and schemas + * + * @enum {number} + */ +export enum MetadataVersion { + /** + * 0.1.0 (October 2016). + */ + V1 = 0, + + /** + * 0.2.0 (February 2017). Non-backwards compatible with V1. + */ + V2 = 1, + + /** + * 0.3.0 -> 0.7.1 (May - December 2017). Non-backwards compatible with V2. + */ + V3 = 2, + + /** + * >= 0.8.0 (December 2017). Non-backwards compatible with V3. + */ + V4 = 3, + + /** + * >= 1.0.0 (July 2020. Backwards compatible with V4 (V5 readers can read V4 + * metadata and IPC messages). Implementations are recommended to provide a + * V4 compatibility mode with V5 format changes disabled. + * + * Incompatible changes between V4 and V5: + * - Union buffer layout has changed. In V5, Unions don't have a validity + * bitmap buffer. + */ + V5 = 4 +} + +/** + * Represents Arrow Features that might not have full support + * within implementations. This is intended to be used in + * two scenarios: + * 1. A mechanism for readers of Arrow Streams + * and files to understand that the stream or file makes + * use of a feature that isn't supported or unknown to + * the implementation (and therefore can meet the Arrow + * forward compatibility guarantees). + * 2. A means of negotiating between a client and server + * what features a stream is allowed to use. The enums + * values here are intented to represent higher level + * features, additional details maybe negotiated + * with key-value pairs specific to the protocol. + * + * Enums added to this list should be assigned power-of-two values + * to facilitate exchanging and comparing bitmaps for supported + * features. + * + * @enum {number} + */ +export enum Feature { + /** + * Needed to make flatbuffers happy. + */ + UNUSED = 0, + + /** + * The stream makes use of multiple full dictionaries with the + * same ID and assumes clients implement dictionary replacement + * correctly. + */ + DICTIONARY_REPLACEMENT = 1, + + /** + * The stream makes use of compressed bodies as described + * in Message.fbs. + */ + COMPRESSED_BODY = 2 +} + +/** + * @enum {number} + */ +export enum UnionMode { + Sparse = 0, + Dense = 1 +} + +/** + * @enum {number} + */ +export enum Precision { + HALF = 0, + SINGLE = 1, + DOUBLE = 2 +} + +/** + * @enum {number} + */ +export enum DateUnit { + DAY = 0, + MILLISECOND = 1 +} + +/** + * @enum {number} + */ +export enum TimeUnit { + SECOND = 0, + MILLISECOND = 1, + MICROSECOND = 2, + NANOSECOND = 3 +} + +/** + * @enum {number} + */ +export enum IntervalUnit { + YEAR_MONTH = 0, + DAY_TIME = 1 +} + +/** + * ---------------------------------------------------------------------- + * Top-level Type value, enabling extensible type-specific metadata. We can + * add new logical types to Type without breaking backwards compatibility + * + * @enum {number} + */ +export enum Type { + NONE = 0, + Null = 1, + Int = 2, + FloatingPoint = 3, + Binary = 4, + Utf8 = 5, + Bool = 6, + Decimal = 7, + Date = 8, + Time = 9, + Timestamp = 10, + Interval = 11, + List = 12, + Struct_ = 13, + Union = 14, + FixedSizeBinary = 15, + FixedSizeList = 16, + Map = 17, + Duration = 18, + LargeBinary = 19, + LargeUtf8 = 20, + LargeList = 21 +} + +/** + * ---------------------------------------------------------------------- + * Dictionary encoding metadata + * Maintained for forwards compatibility, in the future + * Dictionaries might be explicit maps between integers and values + * allowing for non-contiguous index values + * + * @enum {number} + */ +export enum DictionaryKind { + DenseArray = 0 +} + +/** + * ---------------------------------------------------------------------- + * Endianness of the platform producing the data + * + * @enum {number} + */ +export enum Endianness { + Little = 0, + Big = 1 +} + +/** + * These are stored in the flatbuffer in the Type union below + * + * @constructor + */ +export class Null { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Null + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Null { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Null= obj + * @returns Null + */ + static getRootAsNull(bb: flatbuffers.ByteBuffer, obj?: Null): Null { + return (obj || new Null()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Null= obj + * @returns Null + */ + static getSizePrefixedRootAsNull(bb: flatbuffers.ByteBuffer, obj?: Null): Null { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Null()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.Builder builder + */ + static startNull(builder: flatbuffers.Builder) { + builder.startObject(0); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endNull(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createNull(builder: flatbuffers.Builder): flatbuffers.Offset { + Null.startNull(builder); + return Null.endNull(builder); + } +} +/** + * A Struct_ in the flatbuffer metadata is the same as an Arrow Struct + * (according to the physical memory layout). We used Struct_ here as + * Struct is a reserved word in Flatbuffers + * + * @constructor + */ +export class Struct_ { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Struct_ + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Struct_ { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Struct_= obj + * @returns Struct_ + */ + static getRootAsStruct_(bb: flatbuffers.ByteBuffer, obj?: Struct_): Struct_ { + return (obj || new Struct_()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Struct_= obj + * @returns Struct_ + */ + static getSizePrefixedRootAsStruct_(bb: flatbuffers.ByteBuffer, obj?: Struct_): Struct_ { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Struct_()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.Builder builder + */ + static startStruct_(builder: flatbuffers.Builder) { + builder.startObject(0); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endStruct_(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createStruct_(builder: flatbuffers.Builder): flatbuffers.Offset { + Struct_.startStruct_(builder); + return Struct_.endStruct_(builder); + } +} +/** + * @constructor + */ +export class List { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns List + */ + __init(i: number, bb: flatbuffers.ByteBuffer): List { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param List= obj + * @returns List + */ + static getRootAsList(bb: flatbuffers.ByteBuffer, obj?: List): List { + return (obj || new List()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param List= obj + * @returns List + */ + static getSizePrefixedRootAsList(bb: flatbuffers.ByteBuffer, obj?: List): List { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new List()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.Builder builder + */ + static startList(builder: flatbuffers.Builder) { + builder.startObject(0); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endList(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createList(builder: flatbuffers.Builder): flatbuffers.Offset { + List.startList(builder); + return List.endList(builder); + } +} +/** + * Same as List, but with 64-bit offsets, allowing to represent + * extremely large data values. + * + * @constructor + */ +export class LargeList { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns LargeList + */ + __init(i: number, bb: flatbuffers.ByteBuffer): LargeList { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param LargeList= obj + * @returns LargeList + */ + static getRootAsLargeList(bb: flatbuffers.ByteBuffer, obj?: LargeList): LargeList { + return (obj || new LargeList()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param LargeList= obj + * @returns LargeList + */ + static getSizePrefixedRootAsLargeList(bb: flatbuffers.ByteBuffer, obj?: LargeList): LargeList { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new LargeList()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.Builder builder + */ + static startLargeList(builder: flatbuffers.Builder) { + builder.startObject(0); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endLargeList(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createLargeList(builder: flatbuffers.Builder): flatbuffers.Offset { + LargeList.startLargeList(builder); + return LargeList.endLargeList(builder); + } +} +/** + * @constructor + */ +export class FixedSizeList { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns FixedSizeList + */ + __init(i: number, bb: flatbuffers.ByteBuffer): FixedSizeList { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param FixedSizeList= obj + * @returns FixedSizeList + */ + static getRootAsFixedSizeList(bb: flatbuffers.ByteBuffer, obj?: FixedSizeList): FixedSizeList { + return (obj || new FixedSizeList()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param FixedSizeList= obj + * @returns FixedSizeList + */ + static getSizePrefixedRootAsFixedSizeList(bb: flatbuffers.ByteBuffer, obj?: FixedSizeList): FixedSizeList { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new FixedSizeList()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * Number of list items per value + * + * @returns number + */ + listSize(): number { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? this.bb!.readInt32(this.bb_pos + offset) : 0; + } + + /** + * @param flatbuffers.Builder builder + */ + static startFixedSizeList(builder: flatbuffers.Builder) { + builder.startObject(1); + } + + /** + * @param flatbuffers.Builder builder + * @param number listSize + */ + static addListSize(builder: flatbuffers.Builder, listSize: number) { + builder.addFieldInt32(0, listSize, 0); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endFixedSizeList(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createFixedSizeList(builder: flatbuffers.Builder, listSize: number): flatbuffers.Offset { + FixedSizeList.startFixedSizeList(builder); + FixedSizeList.addListSize(builder, listSize); + return FixedSizeList.endFixedSizeList(builder); + } +} +/** + * A Map is a logical nested type that is represented as + * + * List<entries: Struct<key: K, value: V>> + * + * In this layout, the keys and values are each respectively contiguous. We do + * not constrain the key and value types, so the application is responsible + * for ensuring that the keys are hashable and unique. Whether the keys are sorted + * may be set in the metadata for this field. + * + * In a field with Map type, the field has a child Struct field, which then + * has two children: key type and the second the value type. The names of the + * child fields may be respectively "entries", "key", and "value", but this is + * not enforced. + * + * Map + * - child[0] entries: Struct + * - child[0] key: K + * - child[1] value: V + * + * Neither the "entries" field nor the "key" field may be nullable. + * + * The metadata is structured so that Arrow systems without special handling + * for Map can make Map an alias for List. The "layout" attribute for the Map + * field must have the same contents as a List. + * + * @constructor + */ +export class Map { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Map + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Map { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Map= obj + * @returns Map + */ + static getRootAsMap(bb: flatbuffers.ByteBuffer, obj?: Map): Map { + return (obj || new Map()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Map= obj + * @returns Map + */ + static getSizePrefixedRootAsMap(bb: flatbuffers.ByteBuffer, obj?: Map): Map { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Map()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * Set to true if the keys within each value are sorted + * + * @returns boolean + */ + keysSorted(): boolean { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? !!this.bb!.readInt8(this.bb_pos + offset) : false; + } + + /** + * @param flatbuffers.Builder builder + */ + static startMap(builder: flatbuffers.Builder) { + builder.startObject(1); + } + + /** + * @param flatbuffers.Builder builder + * @param boolean keysSorted + */ + static addKeysSorted(builder: flatbuffers.Builder, keysSorted: boolean) { + builder.addFieldInt8(0, +keysSorted, +false); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endMap(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createMap(builder: flatbuffers.Builder, keysSorted: boolean): flatbuffers.Offset { + Map.startMap(builder); + Map.addKeysSorted(builder, keysSorted); + return Map.endMap(builder); + } +} +/** + * A union is a complex type with children in Field + * By default ids in the type vector refer to the offsets in the children + * optionally typeIds provides an indirection between the child offset and the type id + * for each child typeIds[offset] is the id used in the type vector + * + * @constructor + */ +export class Union { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Union + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Union { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Union= obj + * @returns Union + */ + static getRootAsUnion(bb: flatbuffers.ByteBuffer, obj?: Union): Union { + return (obj || new Union()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Union= obj + * @returns Union + */ + static getSizePrefixedRootAsUnion(bb: flatbuffers.ByteBuffer, obj?: Union): Union { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Union()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @returns UnionMode + */ + mode(): UnionMode { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : UnionMode.Sparse; + } + + /** + * @param number index + * @returns number + */ + typeIds(index: number): number | null { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? this.bb!.readInt32(this.bb!.__vector(this.bb_pos + offset) + index * 4) : 0; + } + + /** + * @returns number + */ + typeIdsLength(): number { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; + } + + /** + * @returns Int32Array + */ + typeIdsArray(): Int32Array | null { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? new Int32Array(this.bb!.bytes().buffer, this.bb!.bytes().byteOffset + this.bb!.__vector(this.bb_pos + offset), this.bb!.__vector_len(this.bb_pos + offset)) : null; + } + + /** + * @param flatbuffers.Builder builder + */ + static startUnion(builder: flatbuffers.Builder) { + builder.startObject(2); + } + + /** + * @param flatbuffers.Builder builder + * @param UnionMode mode + */ + static addMode(builder: flatbuffers.Builder, mode: UnionMode) { + builder.addFieldInt16(0, mode, UnionMode.Sparse); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset typeIdsOffset + */ + static addTypeIds(builder: flatbuffers.Builder, typeIdsOffset: flatbuffers.Offset) { + builder.addFieldOffset(1, typeIdsOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param Array.<number> data + * @returns flatbuffers.Offset + */ + static createTypeIdsVector(builder: flatbuffers.Builder, data: number[] | Int32Array): flatbuffers.Offset { + builder.startVector(4, data.length, 4); + for (let i = data.length - 1; i >= 0; i--) { + builder.addInt32(data[i]); + } + return builder.endVector(); + } + + /** + * @param flatbuffers.Builder builder + * @param number numElems + */ + static startTypeIdsVector(builder: flatbuffers.Builder, numElems: number) { + builder.startVector(4, numElems, 4); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endUnion(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createUnion(builder: flatbuffers.Builder, mode: UnionMode, typeIdsOffset: flatbuffers.Offset): flatbuffers.Offset { + Union.startUnion(builder); + Union.addMode(builder, mode); + Union.addTypeIds(builder, typeIdsOffset); + return Union.endUnion(builder); + } +} +/** + * @constructor + */ +export class Int { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Int + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Int { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Int= obj + * @returns Int + */ + static getRootAsInt(bb: flatbuffers.ByteBuffer, obj?: Int): Int { + return (obj || new Int()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Int= obj + * @returns Int + */ + static getSizePrefixedRootAsInt(bb: flatbuffers.ByteBuffer, obj?: Int): Int { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Int()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @returns number + */ + bitWidth(): number { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? this.bb!.readInt32(this.bb_pos + offset) : 0; + } + + /** + * @returns boolean + */ + isSigned(): boolean { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? !!this.bb!.readInt8(this.bb_pos + offset) : false; + } + + /** + * @param flatbuffers.Builder builder + */ + static startInt(builder: flatbuffers.Builder) { + builder.startObject(2); + } + + /** + * @param flatbuffers.Builder builder + * @param number bitWidth + */ + static addBitWidth(builder: flatbuffers.Builder, bitWidth: number) { + builder.addFieldInt32(0, bitWidth, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param boolean isSigned + */ + static addIsSigned(builder: flatbuffers.Builder, isSigned: boolean) { + builder.addFieldInt8(1, +isSigned, +false); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endInt(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createInt(builder: flatbuffers.Builder, bitWidth: number, isSigned: boolean): flatbuffers.Offset { + Int.startInt(builder); + Int.addBitWidth(builder, bitWidth); + Int.addIsSigned(builder, isSigned); + return Int.endInt(builder); + } +} +/** + * @constructor + */ +export class FloatingPoint { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns FloatingPoint + */ + __init(i: number, bb: flatbuffers.ByteBuffer): FloatingPoint { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param FloatingPoint= obj + * @returns FloatingPoint + */ + static getRootAsFloatingPoint(bb: flatbuffers.ByteBuffer, obj?: FloatingPoint): FloatingPoint { + return (obj || new FloatingPoint()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param FloatingPoint= obj + * @returns FloatingPoint + */ + static getSizePrefixedRootAsFloatingPoint(bb: flatbuffers.ByteBuffer, obj?: FloatingPoint): FloatingPoint { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new FloatingPoint()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @returns Precision + */ + precision(): Precision { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : Precision.HALF; + } + + /** + * @param flatbuffers.Builder builder + */ + static startFloatingPoint(builder: flatbuffers.Builder) { + builder.startObject(1); + } + + /** + * @param flatbuffers.Builder builder + * @param Precision precision + */ + static addPrecision(builder: flatbuffers.Builder, precision: Precision) { + builder.addFieldInt16(0, precision, Precision.HALF); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endFloatingPoint(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createFloatingPoint(builder: flatbuffers.Builder, precision: Precision): flatbuffers.Offset { + FloatingPoint.startFloatingPoint(builder); + FloatingPoint.addPrecision(builder, precision); + return FloatingPoint.endFloatingPoint(builder); + } +} +/** + * Unicode with UTF-8 encoding + * + * @constructor + */ +export class Utf8 { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Utf8 + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Utf8 { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Utf8= obj + * @returns Utf8 + */ + static getRootAsUtf8(bb: flatbuffers.ByteBuffer, obj?: Utf8): Utf8 { + return (obj || new Utf8()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Utf8= obj + * @returns Utf8 + */ + static getSizePrefixedRootAsUtf8(bb: flatbuffers.ByteBuffer, obj?: Utf8): Utf8 { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Utf8()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.Builder builder + */ + static startUtf8(builder: flatbuffers.Builder) { + builder.startObject(0); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endUtf8(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createUtf8(builder: flatbuffers.Builder): flatbuffers.Offset { + Utf8.startUtf8(builder); + return Utf8.endUtf8(builder); + } +} +/** + * Opaque binary data + * + * @constructor + */ +export class Binary { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Binary + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Binary { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Binary= obj + * @returns Binary + */ + static getRootAsBinary(bb: flatbuffers.ByteBuffer, obj?: Binary): Binary { + return (obj || new Binary()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Binary= obj + * @returns Binary + */ + static getSizePrefixedRootAsBinary(bb: flatbuffers.ByteBuffer, obj?: Binary): Binary { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Binary()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.Builder builder + */ + static startBinary(builder: flatbuffers.Builder) { + builder.startObject(0); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endBinary(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createBinary(builder: flatbuffers.Builder): flatbuffers.Offset { + Binary.startBinary(builder); + return Binary.endBinary(builder); + } +} +/** + * Same as Utf8, but with 64-bit offsets, allowing to represent + * extremely large data values. + * + * @constructor + */ +export class LargeUtf8 { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns LargeUtf8 + */ + __init(i: number, bb: flatbuffers.ByteBuffer): LargeUtf8 { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param LargeUtf8= obj + * @returns LargeUtf8 + */ + static getRootAsLargeUtf8(bb: flatbuffers.ByteBuffer, obj?: LargeUtf8): LargeUtf8 { + return (obj || new LargeUtf8()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param LargeUtf8= obj + * @returns LargeUtf8 + */ + static getSizePrefixedRootAsLargeUtf8(bb: flatbuffers.ByteBuffer, obj?: LargeUtf8): LargeUtf8 { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new LargeUtf8()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.Builder builder + */ + static startLargeUtf8(builder: flatbuffers.Builder) { + builder.startObject(0); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endLargeUtf8(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createLargeUtf8(builder: flatbuffers.Builder): flatbuffers.Offset { + LargeUtf8.startLargeUtf8(builder); + return LargeUtf8.endLargeUtf8(builder); + } +} +/** + * Same as Binary, but with 64-bit offsets, allowing to represent + * extremely large data values. + * + * @constructor + */ +export class LargeBinary { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns LargeBinary + */ + __init(i: number, bb: flatbuffers.ByteBuffer): LargeBinary { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param LargeBinary= obj + * @returns LargeBinary + */ + static getRootAsLargeBinary(bb: flatbuffers.ByteBuffer, obj?: LargeBinary): LargeBinary { + return (obj || new LargeBinary()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param LargeBinary= obj + * @returns LargeBinary + */ + static getSizePrefixedRootAsLargeBinary(bb: flatbuffers.ByteBuffer, obj?: LargeBinary): LargeBinary { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new LargeBinary()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.Builder builder + */ + static startLargeBinary(builder: flatbuffers.Builder) { + builder.startObject(0); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endLargeBinary(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createLargeBinary(builder: flatbuffers.Builder): flatbuffers.Offset { + LargeBinary.startLargeBinary(builder); + return LargeBinary.endLargeBinary(builder); + } +} +/** + * @constructor + */ +export class FixedSizeBinary { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns FixedSizeBinary + */ + __init(i: number, bb: flatbuffers.ByteBuffer): FixedSizeBinary { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param FixedSizeBinary= obj + * @returns FixedSizeBinary + */ + static getRootAsFixedSizeBinary(bb: flatbuffers.ByteBuffer, obj?: FixedSizeBinary): FixedSizeBinary { + return (obj || new FixedSizeBinary()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param FixedSizeBinary= obj + * @returns FixedSizeBinary + */ + static getSizePrefixedRootAsFixedSizeBinary(bb: flatbuffers.ByteBuffer, obj?: FixedSizeBinary): FixedSizeBinary { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new FixedSizeBinary()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * Number of bytes per value + * + * @returns number + */ + byteWidth(): number { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? this.bb!.readInt32(this.bb_pos + offset) : 0; + } + + /** + * @param flatbuffers.Builder builder + */ + static startFixedSizeBinary(builder: flatbuffers.Builder) { + builder.startObject(1); + } + + /** + * @param flatbuffers.Builder builder + * @param number byteWidth + */ + static addByteWidth(builder: flatbuffers.Builder, byteWidth: number) { + builder.addFieldInt32(0, byteWidth, 0); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endFixedSizeBinary(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createFixedSizeBinary(builder: flatbuffers.Builder, byteWidth: number): flatbuffers.Offset { + FixedSizeBinary.startFixedSizeBinary(builder); + FixedSizeBinary.addByteWidth(builder, byteWidth); + return FixedSizeBinary.endFixedSizeBinary(builder); + } +} +/** + * @constructor + */ +export class Bool { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Bool + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Bool { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Bool= obj + * @returns Bool + */ + static getRootAsBool(bb: flatbuffers.ByteBuffer, obj?: Bool): Bool { + return (obj || new Bool()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Bool= obj + * @returns Bool + */ + static getSizePrefixedRootAsBool(bb: flatbuffers.ByteBuffer, obj?: Bool): Bool { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Bool()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.Builder builder + */ + static startBool(builder: flatbuffers.Builder) { + builder.startObject(0); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endBool(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createBool(builder: flatbuffers.Builder): flatbuffers.Offset { + Bool.startBool(builder); + return Bool.endBool(builder); + } +} +/** + * Exact decimal value represented as an integer value in two's + * complement. Currently only 128-bit (16-byte) and 256-bit (32-byte) integers + * are used. The representation uses the endianness indicated + * in the Schema. + * + * @constructor + */ +export class Decimal { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Decimal + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Decimal { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Decimal= obj + * @returns Decimal + */ + static getRootAsDecimal(bb: flatbuffers.ByteBuffer, obj?: Decimal): Decimal { + return (obj || new Decimal()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Decimal= obj + * @returns Decimal + */ + static getSizePrefixedRootAsDecimal(bb: flatbuffers.ByteBuffer, obj?: Decimal): Decimal { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Decimal()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * Total number of decimal digits + * + * @returns number + */ + precision(): number { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? this.bb!.readInt32(this.bb_pos + offset) : 0; + } + + /** + * Number of digits after the decimal point "." + * + * @returns number + */ + scale(): number { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? this.bb!.readInt32(this.bb_pos + offset) : 0; + } + + /** + * Number of bits per value. The only accepted widths are 128 and 256. + * We use bitWidth for consistency with Int::bitWidth. + * + * @returns number + */ + bitWidth(): number { + const offset = this.bb!.__offset(this.bb_pos, 8); + return offset ? this.bb!.readInt32(this.bb_pos + offset) : 128; + } + + /** + * @param flatbuffers.Builder builder + */ + static startDecimal(builder: flatbuffers.Builder) { + builder.startObject(3); + } + + /** + * @param flatbuffers.Builder builder + * @param number precision + */ + static addPrecision(builder: flatbuffers.Builder, precision: number) { + builder.addFieldInt32(0, precision, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param number scale + */ + static addScale(builder: flatbuffers.Builder, scale: number) { + builder.addFieldInt32(1, scale, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param number bitWidth + */ + static addBitWidth(builder: flatbuffers.Builder, bitWidth: number) { + builder.addFieldInt32(2, bitWidth, 128); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endDecimal(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createDecimal(builder: flatbuffers.Builder, precision: number, scale: number, bitWidth: number): flatbuffers.Offset { + Decimal.startDecimal(builder); + Decimal.addPrecision(builder, precision); + Decimal.addScale(builder, scale); + Decimal.addBitWidth(builder, bitWidth); + return Decimal.endDecimal(builder); + } +} +/** + * Date is either a 32-bit or 64-bit type representing elapsed time since UNIX + * epoch (1970-01-01), stored in either of two units: + * + * * Milliseconds (64 bits) indicating UNIX time elapsed since the epoch (no + * leap seconds), where the values are evenly divisible by 86400000 + * * Days (32 bits) since the UNIX epoch + * + * @constructor + */ +export class Date { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Date + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Date { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Date= obj + * @returns Date + */ + static getRootAsDate(bb: flatbuffers.ByteBuffer, obj?: Date): Date { + return (obj || new Date()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Date= obj + * @returns Date + */ + static getSizePrefixedRootAsDate(bb: flatbuffers.ByteBuffer, obj?: Date): Date { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Date()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @returns DateUnit + */ + unit(): DateUnit { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : DateUnit.MILLISECOND; + } + + /** + * @param flatbuffers.Builder builder + */ + static startDate(builder: flatbuffers.Builder) { + builder.startObject(1); + } + + /** + * @param flatbuffers.Builder builder + * @param DateUnit unit + */ + static addUnit(builder: flatbuffers.Builder, unit: DateUnit) { + builder.addFieldInt16(0, unit, DateUnit.MILLISECOND); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endDate(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createDate(builder: flatbuffers.Builder, unit: DateUnit): flatbuffers.Offset { + Date.startDate(builder); + Date.addUnit(builder, unit); + return Date.endDate(builder); + } +} +/** + * Time type. The physical storage type depends on the unit + * - SECOND and MILLISECOND: 32 bits + * - MICROSECOND and NANOSECOND: 64 bits + * + * @constructor + */ +export class Time { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Time + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Time { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Time= obj + * @returns Time + */ + static getRootAsTime(bb: flatbuffers.ByteBuffer, obj?: Time): Time { + return (obj || new Time()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Time= obj + * @returns Time + */ + static getSizePrefixedRootAsTime(bb: flatbuffers.ByteBuffer, obj?: Time): Time { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Time()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @returns TimeUnit + */ + unit(): TimeUnit { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : TimeUnit.MILLISECOND; + } + + /** + * @returns number + */ + bitWidth(): number { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? this.bb!.readInt32(this.bb_pos + offset) : 32; + } + + /** + * @param flatbuffers.Builder builder + */ + static startTime(builder: flatbuffers.Builder) { + builder.startObject(2); + } + + /** + * @param flatbuffers.Builder builder + * @param TimeUnit unit + */ + static addUnit(builder: flatbuffers.Builder, unit: TimeUnit) { + builder.addFieldInt16(0, unit, TimeUnit.MILLISECOND); + } + + /** + * @param flatbuffers.Builder builder + * @param number bitWidth + */ + static addBitWidth(builder: flatbuffers.Builder, bitWidth: number) { + builder.addFieldInt32(1, bitWidth, 32); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endTime(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createTime(builder: flatbuffers.Builder, unit: TimeUnit, bitWidth: number): flatbuffers.Offset { + Time.startTime(builder); + Time.addUnit(builder, unit); + Time.addBitWidth(builder, bitWidth); + return Time.endTime(builder); + } +} +/** + * Time elapsed from the Unix epoch, 00:00:00.000 on 1 January 1970, excluding + * leap seconds, as a 64-bit integer. Note that UNIX time does not include + * leap seconds. + * + * The Timestamp metadata supports both "time zone naive" and "time zone + * aware" timestamps. Read about the timezone attribute for more detail + * + * @constructor + */ +export class Timestamp { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Timestamp + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Timestamp { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Timestamp= obj + * @returns Timestamp + */ + static getRootAsTimestamp(bb: flatbuffers.ByteBuffer, obj?: Timestamp): Timestamp { + return (obj || new Timestamp()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Timestamp= obj + * @returns Timestamp + */ + static getSizePrefixedRootAsTimestamp(bb: flatbuffers.ByteBuffer, obj?: Timestamp): Timestamp { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Timestamp()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @returns TimeUnit + */ + unit(): TimeUnit { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : TimeUnit.SECOND; + } + + /** + * The time zone is a string indicating the name of a time zone, one of: + * + * * As used in the Olson time zone database (the "tz database" or + * "tzdata"), such as "America/New_York" + * * An absolute time zone offset of the form +XX:XX or -XX:XX, such as +07:30 + * + * Whether a timezone string is present indicates different semantics about + * the data: + * + * * If the time zone is null or equal to an empty string, the data is "time + * zone naive" and shall be displayed *as is* to the user, not localized + * to the locale of the user. This data can be though of as UTC but + * without having "UTC" as the time zone, it is not considered to be + * localized to any time zone + * + * * If the time zone is set to a valid value, values can be displayed as + * "localized" to that time zone, even though the underlying 64-bit + * integers are identical to the same data stored in UTC. Converting + * between time zones is a metadata-only operation and does not change the + * underlying values + * + * @param flatbuffers.Encoding= optionalEncoding + * @returns string|Uint8Array|null + */ + timezone(): string | null; + timezone(optionalEncoding: flatbuffers.Encoding): string | Uint8Array | null; + timezone(optionalEncoding?: any): string | Uint8Array | null { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null; + } + + /** + * @param flatbuffers.Builder builder + */ + static startTimestamp(builder: flatbuffers.Builder) { + builder.startObject(2); + } + + /** + * @param flatbuffers.Builder builder + * @param TimeUnit unit + */ + static addUnit(builder: flatbuffers.Builder, unit: TimeUnit) { + builder.addFieldInt16(0, unit, TimeUnit.SECOND); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset timezoneOffset + */ + static addTimezone(builder: flatbuffers.Builder, timezoneOffset: flatbuffers.Offset) { + builder.addFieldOffset(1, timezoneOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endTimestamp(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createTimestamp(builder: flatbuffers.Builder, unit: TimeUnit, timezoneOffset: flatbuffers.Offset): flatbuffers.Offset { + Timestamp.startTimestamp(builder); + Timestamp.addUnit(builder, unit); + Timestamp.addTimezone(builder, timezoneOffset); + return Timestamp.endTimestamp(builder); + } +} +/** + * @constructor + */ +export class Interval { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Interval + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Interval { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Interval= obj + * @returns Interval + */ + static getRootAsInterval(bb: flatbuffers.ByteBuffer, obj?: Interval): Interval { + return (obj || new Interval()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Interval= obj + * @returns Interval + */ + static getSizePrefixedRootAsInterval(bb: flatbuffers.ByteBuffer, obj?: Interval): Interval { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Interval()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @returns IntervalUnit + */ + unit(): IntervalUnit { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : IntervalUnit.YEAR_MONTH; + } + + /** + * @param flatbuffers.Builder builder + */ + static startInterval(builder: flatbuffers.Builder) { + builder.startObject(1); + } + + /** + * @param flatbuffers.Builder builder + * @param IntervalUnit unit + */ + static addUnit(builder: flatbuffers.Builder, unit: IntervalUnit) { + builder.addFieldInt16(0, unit, IntervalUnit.YEAR_MONTH); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endInterval(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createInterval(builder: flatbuffers.Builder, unit: IntervalUnit): flatbuffers.Offset { + Interval.startInterval(builder); + Interval.addUnit(builder, unit); + return Interval.endInterval(builder); + } +} +/** + * @constructor + */ +export class Duration { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Duration + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Duration { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Duration= obj + * @returns Duration + */ + static getRootAsDuration(bb: flatbuffers.ByteBuffer, obj?: Duration): Duration { + return (obj || new Duration()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Duration= obj + * @returns Duration + */ + static getSizePrefixedRootAsDuration(bb: flatbuffers.ByteBuffer, obj?: Duration): Duration { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Duration()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @returns TimeUnit + */ + unit(): TimeUnit { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : TimeUnit.MILLISECOND; + } + + /** + * @param flatbuffers.Builder builder + */ + static startDuration(builder: flatbuffers.Builder) { + builder.startObject(1); + } + + /** + * @param flatbuffers.Builder builder + * @param TimeUnit unit + */ + static addUnit(builder: flatbuffers.Builder, unit: TimeUnit) { + builder.addFieldInt16(0, unit, TimeUnit.MILLISECOND); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endDuration(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createDuration(builder: flatbuffers.Builder, unit: TimeUnit): flatbuffers.Offset { + Duration.startDuration(builder); + Duration.addUnit(builder, unit); + return Duration.endDuration(builder); + } +} +/** + * ---------------------------------------------------------------------- + * user defined key value pairs to add custom metadata to arrow + * key namespacing is the responsibility of the user + * + * @constructor + */ +export class KeyValue { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns KeyValue + */ + __init(i: number, bb: flatbuffers.ByteBuffer): KeyValue { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param KeyValue= obj + * @returns KeyValue + */ + static getRootAsKeyValue(bb: flatbuffers.ByteBuffer, obj?: KeyValue): KeyValue { + return (obj || new KeyValue()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param KeyValue= obj + * @returns KeyValue + */ + static getSizePrefixedRootAsKeyValue(bb: flatbuffers.ByteBuffer, obj?: KeyValue): KeyValue { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new KeyValue()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.Encoding= optionalEncoding + * @returns string|Uint8Array|null + */ + key(): string | null; + key(optionalEncoding: flatbuffers.Encoding): string | Uint8Array | null; + key(optionalEncoding?: any): string | Uint8Array | null { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null; + } + + /** + * @param flatbuffers.Encoding= optionalEncoding + * @returns string|Uint8Array|null + */ + value(): string | null; + value(optionalEncoding: flatbuffers.Encoding): string | Uint8Array | null; + value(optionalEncoding?: any): string | Uint8Array | null { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null; + } + + /** + * @param flatbuffers.Builder builder + */ + static startKeyValue(builder: flatbuffers.Builder) { + builder.startObject(2); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset keyOffset + */ + static addKey(builder: flatbuffers.Builder, keyOffset: flatbuffers.Offset) { + builder.addFieldOffset(0, keyOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset valueOffset + */ + static addValue(builder: flatbuffers.Builder, valueOffset: flatbuffers.Offset) { + builder.addFieldOffset(1, valueOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endKeyValue(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createKeyValue(builder: flatbuffers.Builder, keyOffset: flatbuffers.Offset, valueOffset: flatbuffers.Offset): flatbuffers.Offset { + KeyValue.startKeyValue(builder); + KeyValue.addKey(builder, keyOffset); + KeyValue.addValue(builder, valueOffset); + return KeyValue.endKeyValue(builder); + } +} +/** + * @constructor + */ +export class DictionaryEncoding { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns DictionaryEncoding + */ + __init(i: number, bb: flatbuffers.ByteBuffer): DictionaryEncoding { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param DictionaryEncoding= obj + * @returns DictionaryEncoding + */ + static getRootAsDictionaryEncoding(bb: flatbuffers.ByteBuffer, obj?: DictionaryEncoding): DictionaryEncoding { + return (obj || new DictionaryEncoding()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param DictionaryEncoding= obj + * @returns DictionaryEncoding + */ + static getSizePrefixedRootAsDictionaryEncoding(bb: flatbuffers.ByteBuffer, obj?: DictionaryEncoding): DictionaryEncoding { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new DictionaryEncoding()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * The known dictionary id in the application where this data is used. In + * the file or streaming formats, the dictionary ids are found in the + * DictionaryBatch messages + * + * @returns flatbuffers.Long + */ + id(): flatbuffers.Long { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? this.bb!.readInt64(this.bb_pos + offset) : this.bb!.createLong(0, 0); + } + + /** + * The dictionary indices are constrained to be non-negative integers. If + * this field is null, the indices must be signed int32. To maximize + * cross-language compatibility and performance, implementations are + * recommended to prefer signed integer types over unsigned integer types + * and to avoid uint64 indices unless they are required by an application. + * + * @param Int= obj + * @returns Int|null + */ + indexType(obj?: Int): Int | null { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? (obj || new Int()).__init(this.bb!.__indirect(this.bb_pos + offset), this.bb!) : null; + } + + /** + * By default, dictionaries are not ordered, or the order does not have + * semantic meaning. In some statistical, applications, dictionary-encoding + * is used to represent ordered categorical data, and we provide a way to + * preserve that metadata here + * + * @returns boolean + */ + isOrdered(): boolean { + const offset = this.bb!.__offset(this.bb_pos, 8); + return offset ? !!this.bb!.readInt8(this.bb_pos + offset) : false; + } + + /** + * @returns DictionaryKind + */ + dictionaryKind(): DictionaryKind { + const offset = this.bb!.__offset(this.bb_pos, 10); + return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : DictionaryKind.DenseArray; + } + + /** + * @param flatbuffers.Builder builder + */ + static startDictionaryEncoding(builder: flatbuffers.Builder) { + builder.startObject(4); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Long id + */ + static addId(builder: flatbuffers.Builder, id: flatbuffers.Long) { + builder.addFieldInt64(0, id, builder.createLong(0, 0)); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset indexTypeOffset + */ + static addIndexType(builder: flatbuffers.Builder, indexTypeOffset: flatbuffers.Offset) { + builder.addFieldOffset(1, indexTypeOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param boolean isOrdered + */ + static addIsOrdered(builder: flatbuffers.Builder, isOrdered: boolean) { + builder.addFieldInt8(2, +isOrdered, +false); + } + + /** + * @param flatbuffers.Builder builder + * @param DictionaryKind dictionaryKind + */ + static addDictionaryKind(builder: flatbuffers.Builder, dictionaryKind: DictionaryKind) { + builder.addFieldInt16(3, dictionaryKind, DictionaryKind.DenseArray); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endDictionaryEncoding(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createDictionaryEncoding(builder: flatbuffers.Builder, id: flatbuffers.Long, indexTypeOffset: flatbuffers.Offset, isOrdered: boolean, dictionaryKind: DictionaryKind): flatbuffers.Offset { + DictionaryEncoding.startDictionaryEncoding(builder); + DictionaryEncoding.addId(builder, id); + DictionaryEncoding.addIndexType(builder, indexTypeOffset); + DictionaryEncoding.addIsOrdered(builder, isOrdered); + DictionaryEncoding.addDictionaryKind(builder, dictionaryKind); + return DictionaryEncoding.endDictionaryEncoding(builder); + } +} +/** + * ---------------------------------------------------------------------- + * A field represents a named column in a record / row batch or child of a + * nested type. + * + * @constructor + */ +export class Field { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Field + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Field { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Field= obj + * @returns Field + */ + static getRootAsField(bb: flatbuffers.ByteBuffer, obj?: Field): Field { + return (obj || new Field()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Field= obj + * @returns Field + */ + static getSizePrefixedRootAsField(bb: flatbuffers.ByteBuffer, obj?: Field): Field { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Field()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * Name is not required, in i.e. a List + * + * @param flatbuffers.Encoding= optionalEncoding + * @returns string|Uint8Array|null + */ + name(): string | null; + name(optionalEncoding: flatbuffers.Encoding): string | Uint8Array | null; + name(optionalEncoding?: any): string | Uint8Array | null { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? this.bb!.__string(this.bb_pos + offset, optionalEncoding) : null; + } + + /** + * Whether or not this field can contain nulls. Should be true in general. + * + * @returns boolean + */ + nullable(): boolean { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? !!this.bb!.readInt8(this.bb_pos + offset) : false; + } + + /** + * @returns Type + */ + typeType(): Type { + const offset = this.bb!.__offset(this.bb_pos, 8); + return offset ? /** */ (this.bb!.readUint8(this.bb_pos + offset)) : Type.NONE; + } + + /** + * This is the type of the decoded value if the field is dictionary encoded. + * + * @param flatbuffers.Table obj + * @returns ?flatbuffers.Table + */ + type<T extends flatbuffers.Table>(obj: T): T | null { + const offset = this.bb!.__offset(this.bb_pos, 10); + return offset ? this.bb!.__union(obj, this.bb_pos + offset) : null; + } + + /** + * Present only if the field is dictionary encoded. + * + * @param DictionaryEncoding= obj + * @returns DictionaryEncoding|null + */ + dictionary(obj?: DictionaryEncoding): DictionaryEncoding | null { + const offset = this.bb!.__offset(this.bb_pos, 12); + return offset ? (obj || new DictionaryEncoding()).__init(this.bb!.__indirect(this.bb_pos + offset), this.bb!) : null; + } + + /** + * children apply only to nested data types like Struct, List and Union. For + * primitive types children will have length 0. + * + * @param number index + * @param Field= obj + * @returns Field + */ + children(index: number, obj?: Field): Field | null { + const offset = this.bb!.__offset(this.bb_pos, 14); + return offset ? (obj || new Field()).__init(this.bb!.__indirect(this.bb!.__vector(this.bb_pos + offset) + index * 4), this.bb!) : null; + } + + /** + * @returns number + */ + childrenLength(): number { + const offset = this.bb!.__offset(this.bb_pos, 14); + return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; + } + + /** + * User-defined metadata + * + * @param number index + * @param KeyValue= obj + * @returns KeyValue + */ + customMetadata(index: number, obj?: KeyValue): KeyValue | null { + const offset = this.bb!.__offset(this.bb_pos, 16); + return offset ? (obj || new KeyValue()).__init(this.bb!.__indirect(this.bb!.__vector(this.bb_pos + offset) + index * 4), this.bb!) : null; + } + + /** + * @returns number + */ + customMetadataLength(): number { + const offset = this.bb!.__offset(this.bb_pos, 16); + return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; + } + + /** + * @param flatbuffers.Builder builder + */ + static startField(builder: flatbuffers.Builder) { + builder.startObject(7); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset nameOffset + */ + static addName(builder: flatbuffers.Builder, nameOffset: flatbuffers.Offset) { + builder.addFieldOffset(0, nameOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param boolean nullable + */ + static addNullable(builder: flatbuffers.Builder, nullable: boolean) { + builder.addFieldInt8(1, +nullable, +false); + } + + /** + * @param flatbuffers.Builder builder + * @param Type typeType + */ + static addTypeType(builder: flatbuffers.Builder, typeType: Type) { + builder.addFieldInt8(2, typeType, Type.NONE); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset typeOffset + */ + static addType(builder: flatbuffers.Builder, typeOffset: flatbuffers.Offset) { + builder.addFieldOffset(3, typeOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset dictionaryOffset + */ + static addDictionary(builder: flatbuffers.Builder, dictionaryOffset: flatbuffers.Offset) { + builder.addFieldOffset(4, dictionaryOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset childrenOffset + */ + static addChildren(builder: flatbuffers.Builder, childrenOffset: flatbuffers.Offset) { + builder.addFieldOffset(5, childrenOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param Array.<flatbuffers.Offset> data + * @returns flatbuffers.Offset + */ + static createChildrenVector(builder: flatbuffers.Builder, data: flatbuffers.Offset[]): flatbuffers.Offset { + builder.startVector(4, data.length, 4); + for (let i = data.length - 1; i >= 0; i--) { + builder.addOffset(data[i]); + } + return builder.endVector(); + } + + /** + * @param flatbuffers.Builder builder + * @param number numElems + */ + static startChildrenVector(builder: flatbuffers.Builder, numElems: number) { + builder.startVector(4, numElems, 4); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset customMetadataOffset + */ + static addCustomMetadata(builder: flatbuffers.Builder, customMetadataOffset: flatbuffers.Offset) { + builder.addFieldOffset(6, customMetadataOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param Array.<flatbuffers.Offset> data + * @returns flatbuffers.Offset + */ + static createCustomMetadataVector(builder: flatbuffers.Builder, data: flatbuffers.Offset[]): flatbuffers.Offset { + builder.startVector(4, data.length, 4); + for (let i = data.length - 1; i >= 0; i--) { + builder.addOffset(data[i]); + } + return builder.endVector(); + } + + /** + * @param flatbuffers.Builder builder + * @param number numElems + */ + static startCustomMetadataVector(builder: flatbuffers.Builder, numElems: number) { + builder.startVector(4, numElems, 4); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endField(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + static createField(builder: flatbuffers.Builder, nameOffset: flatbuffers.Offset, nullable: boolean, typeType: Type, typeOffset: flatbuffers.Offset, dictionaryOffset: flatbuffers.Offset, childrenOffset: flatbuffers.Offset, customMetadataOffset: flatbuffers.Offset): flatbuffers.Offset { + Field.startField(builder); + Field.addName(builder, nameOffset); + Field.addNullable(builder, nullable); + Field.addTypeType(builder, typeType); + Field.addType(builder, typeOffset); + Field.addDictionary(builder, dictionaryOffset); + Field.addChildren(builder, childrenOffset); + Field.addCustomMetadata(builder, customMetadataOffset); + return Field.endField(builder); + } +} +/** + * ---------------------------------------------------------------------- + * A Buffer represents a single contiguous memory segment + * + * @constructor + */ +export class Buffer { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Buffer + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Buffer { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * The relative offset into the shared memory page where the bytes for this + * buffer starts + * + * @returns flatbuffers.Long + */ + offset(): flatbuffers.Long { + return this.bb!.readInt64(this.bb_pos); + } + + /** + * The absolute length (in bytes) of the memory buffer. The memory is found + * from offset (inclusive) to offset + length (non-inclusive). When building + * messages using the encapsulated IPC message, padding bytes may be written + * after a buffer, but such padding bytes do not need to be accounted for in + * the size here. + * + * @returns flatbuffers.Long + */ + length(): flatbuffers.Long { + return this.bb!.readInt64(this.bb_pos + 8); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Long offset + * @param flatbuffers.Long length + * @returns flatbuffers.Offset + */ + static createBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Long, length: flatbuffers.Long): flatbuffers.Offset { + builder.prep(8, 16); + builder.writeInt64(length); + builder.writeInt64(offset); + return builder.offset(); + } + +} +/** + * ---------------------------------------------------------------------- + * A Schema describes the columns in a row batch + * + * @constructor + */ +export class Schema { + bb: flatbuffers.ByteBuffer | null = null; + + bb_pos: number = 0; + /** + * @param number i + * @param flatbuffers.ByteBuffer bb + * @returns Schema + */ + __init(i: number, bb: flatbuffers.ByteBuffer): Schema { + this.bb_pos = i; + this.bb = bb; + return this; + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Schema= obj + * @returns Schema + */ + static getRootAsSchema(bb: flatbuffers.ByteBuffer, obj?: Schema): Schema { + return (obj || new Schema()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * @param flatbuffers.ByteBuffer bb + * @param Schema= obj + * @returns Schema + */ + static getSizePrefixedRootAsSchema(bb: flatbuffers.ByteBuffer, obj?: Schema): Schema { + bb.setPosition(bb.position() + flatbuffers.SIZE_PREFIX_LENGTH); + return (obj || new Schema()).__init(bb.readInt32(bb.position()) + bb.position(), bb); + } + + /** + * endianness of the buffer + * it is Little Endian by default + * if endianness doesn't match the underlying system then the vectors need to be converted + * + * @returns Endianness + */ + endianness(): Endianness { + const offset = this.bb!.__offset(this.bb_pos, 4); + return offset ? /** */ (this.bb!.readInt16(this.bb_pos + offset)) : Endianness.Little; + } + + /** + * @param number index + * @param Field= obj + * @returns Field + */ + fields(index: number, obj?: Field): Field | null { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? (obj || new Field()).__init(this.bb!.__indirect(this.bb!.__vector(this.bb_pos + offset) + index * 4), this.bb!) : null; + } + + /** + * @returns number + */ + fieldsLength(): number { + const offset = this.bb!.__offset(this.bb_pos, 6); + return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; + } + + /** + * @param number index + * @param KeyValue= obj + * @returns KeyValue + */ + customMetadata(index: number, obj?: KeyValue): KeyValue | null { + const offset = this.bb!.__offset(this.bb_pos, 8); + return offset ? (obj || new KeyValue()).__init(this.bb!.__indirect(this.bb!.__vector(this.bb_pos + offset) + index * 4), this.bb!) : null; + } + + /** + * @returns number + */ + customMetadataLength(): number { + const offset = this.bb!.__offset(this.bb_pos, 8); + return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; + } + + /** + * Features used in the stream/file. + * + * @param number index + * @returns flatbuffers.Long + */ + features(index: number): flatbuffers.Long | null { + const offset = this.bb!.__offset(this.bb_pos, 10); + return offset ? /** */ (this.bb!.readInt64(this.bb!.__vector(this.bb_pos + offset) + index * 8)) : this.bb!.createLong(0, 0); + } + + /** + * @returns number + */ + featuresLength(): number { + const offset = this.bb!.__offset(this.bb_pos, 10); + return offset ? this.bb!.__vector_len(this.bb_pos + offset) : 0; + } + + /** + * @param flatbuffers.Builder builder + */ + static startSchema(builder: flatbuffers.Builder) { + builder.startObject(4); + } + + /** + * @param flatbuffers.Builder builder + * @param Endianness endianness + */ + static addEndianness(builder: flatbuffers.Builder, endianness: Endianness) { + builder.addFieldInt16(0, endianness, Endianness.Little); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset fieldsOffset + */ + static addFields(builder: flatbuffers.Builder, fieldsOffset: flatbuffers.Offset) { + builder.addFieldOffset(1, fieldsOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param Array.<flatbuffers.Offset> data + * @returns flatbuffers.Offset + */ + static createFieldsVector(builder: flatbuffers.Builder, data: flatbuffers.Offset[]): flatbuffers.Offset { + builder.startVector(4, data.length, 4); + for (let i = data.length - 1; i >= 0; i--) { + builder.addOffset(data[i]); + } + return builder.endVector(); + } + + /** + * @param flatbuffers.Builder builder + * @param number numElems + */ + static startFieldsVector(builder: flatbuffers.Builder, numElems: number) { + builder.startVector(4, numElems, 4); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset customMetadataOffset + */ + static addCustomMetadata(builder: flatbuffers.Builder, customMetadataOffset: flatbuffers.Offset) { + builder.addFieldOffset(2, customMetadataOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param Array.<flatbuffers.Offset> data + * @returns flatbuffers.Offset + */ + static createCustomMetadataVector(builder: flatbuffers.Builder, data: flatbuffers.Offset[]): flatbuffers.Offset { + builder.startVector(4, data.length, 4); + for (let i = data.length - 1; i >= 0; i--) { + builder.addOffset(data[i]); + } + return builder.endVector(); + } + + /** + * @param flatbuffers.Builder builder + * @param number numElems + */ + static startCustomMetadataVector(builder: flatbuffers.Builder, numElems: number) { + builder.startVector(4, numElems, 4); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset featuresOffset + */ + static addFeatures(builder: flatbuffers.Builder, featuresOffset: flatbuffers.Offset) { + builder.addFieldOffset(3, featuresOffset, 0); + } + + /** + * @param flatbuffers.Builder builder + * @param Array.<flatbuffers.Long> data + * @returns flatbuffers.Offset + */ + static createFeaturesVector(builder: flatbuffers.Builder, data: flatbuffers.Long[]): flatbuffers.Offset { + builder.startVector(8, data.length, 8); + for (let i = data.length - 1; i >= 0; i--) { + builder.addInt64(data[i]); + } + return builder.endVector(); + } + + /** + * @param flatbuffers.Builder builder + * @param number numElems + */ + static startFeaturesVector(builder: flatbuffers.Builder, numElems: number) { + builder.startVector(8, numElems, 8); + } + + /** + * @param flatbuffers.Builder builder + * @returns flatbuffers.Offset + */ + static endSchema(builder: flatbuffers.Builder): flatbuffers.Offset { + const offset = builder.endObject(); + return offset; + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset offset + */ + static finishSchemaBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset) { + builder.finish(offset); + } + + /** + * @param flatbuffers.Builder builder + * @param flatbuffers.Offset offset + */ + static finishSizePrefixedSchemaBuffer(builder: flatbuffers.Builder, offset: flatbuffers.Offset) { + builder.finish(offset, undefined, true); + } + + static createSchema(builder: flatbuffers.Builder, endianness: Endianness, fieldsOffset: flatbuffers.Offset, customMetadataOffset: flatbuffers.Offset, featuresOffset: flatbuffers.Offset): flatbuffers.Offset { + Schema.startSchema(builder); + Schema.addEndianness(builder, endianness); + Schema.addFields(builder, fieldsOffset); + Schema.addCustomMetadata(builder, customMetadataOffset); + Schema.addFeatures(builder, featuresOffset); + return Schema.endSchema(builder); + } +} diff --git a/src/arrow/js/src/interfaces.ts b/src/arrow/js/src/interfaces.ts new file mode 100644 index 000000000..43977ca7a --- /dev/null +++ b/src/arrow/js/src/interfaces.ts @@ -0,0 +1,417 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from './data'; +import { Type } from './enum'; +import * as type from './type'; +import { DataType } from './type'; +import * as vecs from './vector/index'; +import * as builders from './builder/index'; +import { BuilderOptions } from './builder/index'; + +/** @ignore */ type FloatArray = Float32Array | Float64Array; +/** @ignore */ type IntArray = Int8Array | Int16Array | Int32Array; +/** @ignore */ type UintArray = Uint8Array | Uint16Array | Uint32Array | Uint8ClampedArray; +/** @ignore */ +export type TypedArray = FloatArray | IntArray | UintArray; +/** @ignore */ +export type BigIntArray = BigInt64Array | BigUint64Array; + +/** @ignore */ +export interface TypedArrayConstructor<T extends TypedArray> { + readonly prototype: T; + new(length?: number): T; + new(array: Iterable<number>): T; + new(buffer: ArrayBufferLike, byteOffset?: number, length?: number): T; + /** + * The size in bytes of each element in the array. + */ + readonly BYTES_PER_ELEMENT: number; + /** + * Returns a new array from a set of elements. + * @param items A set of elements to include in the new array object. + */ + of(...items: number[]): T; + /** + * Creates an array from an array-like or iterable object. + * @param arrayLike An array-like or iterable object to convert to an array. + * @param mapfn A mapping function to call on every element of the array. + * @param thisArg Value of 'this' used to invoke the mapfn. + */ + from(arrayLike: ArrayLike<number>, mapfn?: (v: number, k: number) => number, thisArg?: any): T; + from<U>(arrayLike: ArrayLike<U>, mapfn: (v: U, k: number) => number, thisArg?: any): T; +} + +/** @ignore */ +export interface BigIntArrayConstructor<T extends BigIntArray> { + readonly prototype: T; + new(length?: number): T; + new(array: Iterable<bigint>): T; + new(buffer: ArrayBufferLike, byteOffset?: number, length?: number): T; + /** + * The size in bytes of each element in the array. + */ + readonly BYTES_PER_ELEMENT: number; + /** + * Returns a new array from a set of elements. + * @param items A set of elements to include in the new array object. + */ + of(...items: bigint[]): T; + /** + * Creates an array from an array-like or iterable object. + * @param arrayLike An array-like or iterable object to convert to an array. + * @param mapfn A mapping function to call on every element of the array. + * @param thisArg Value of 'this' used to invoke the mapfn. + */ + from(arrayLike: ArrayLike<bigint>, mapfn?: (v: bigint, k: number) => bigint, thisArg?: any): T; + from<U>(arrayLike: ArrayLike<U>, mapfn: (v: U, k: number) => bigint, thisArg?: any): T; +} + +/** @ignore */ +export type VectorCtorArgs< + T extends VectorType<R>, + R extends DataType = any, + TArgs extends any[] = any[], + TCtor extends new (data: Data<R>, ...args: TArgs) => T = + new (data: Data<R>, ...args: TArgs) => T +> = TCtor extends new (data: Data<R>, ...args: infer TArgs) => T ? TArgs : never; + +/** @ignore */ +export type BuilderCtorArgs< + T extends BuilderType<R, any>, + R extends DataType = any, + TArgs extends any[] = any[], + TCtor extends new (type: R, ...args: TArgs) => T = + new (type: R, ...args: TArgs) => T +> = TCtor extends new (type: R, ...args: infer TArgs) => T ? TArgs : never; + +/** + * Obtain the constructor function of an instance type + * @ignore + */ +export type ConstructorType< + T, + TCtor extends new (...args: any[]) => T = + new (...args: any[]) => T +> = TCtor extends new (...args: any[]) => T ? TCtor : never; + +/** @ignore */ +export type VectorCtorType< + T extends VectorType<R>, + R extends DataType = any, + TCtor extends new (type: R, data?: Data<R>[], offsets?: Uint32Array) => T = + new (type: R, data?: Data<R>[], offsets?: Uint32Array) => T +> = TCtor extends new (type: R, data?: Data<R>[], offsets?: Uint32Array) => T ? TCtor : never; + +/** @ignore */ +export type BuilderCtorType< + T extends BuilderType<R, any>, + R extends DataType = any, + TCtor extends new (options: BuilderOptions<R, any>) => T = + new (options: BuilderOptions<R, any>) => T +> = TCtor extends new (options: BuilderOptions<R, any>) => T ? TCtor : never; + +/** @ignore */ +export type VectorType<T extends Type | DataType = any> = + T extends Type ? TypeToVector<T> : + T extends DataType ? DataTypeToVector<T> : + vecs.BaseVector<any> + ; + +/** @ignore */ +export type BuilderType<T extends Type | DataType = any, TNull = any> = + T extends Type ? TypeToBuilder<T, TNull> : + T extends DataType ? DataTypeToBuilder<T, TNull> : + builders.Builder<any, TNull> + ; + +/** @ignore */ +export type VectorCtor<T extends Type | DataType | VectorType> = + T extends VectorType ? VectorCtorType<VectorType<T['TType']>> : + T extends Type ? VectorCtorType<VectorType<T>> : + T extends DataType ? VectorCtorType<VectorType<T['TType']>> : + VectorCtorType<vecs.BaseVector<any>> + ; + +/** @ignore */ +export type BuilderCtor<T extends Type | DataType = any> = + T extends Type ? BuilderCtorType<BuilderType<T>> : + T extends DataType ? BuilderCtorType<BuilderType<T>> : + BuilderCtorType<builders.Builder> + ; + +/** @ignore */ +export type DataTypeCtor<T extends Type | DataType | VectorType = any> = + T extends DataType ? ConstructorType<T> : + T extends VectorType ? ConstructorType<T['type']> : + T extends Type ? ConstructorType<TypeToDataType<T>> : + never + ; + +/** @ignore */ +export type TypedArrayDataType<T extends Exclude<TypedArray, Uint8ClampedArray> | BigIntArray> = + T extends Int8Array ? type.Int8 : + T extends Int16Array ? type.Int16 : + T extends Int32Array ? type.Int32 : + T extends BigInt64Array ? type.Int64 : + T extends Uint8Array ? type.Uint8 : + T extends Uint16Array ? type.Uint16 : + T extends Uint32Array ? type.Uint32 : + T extends BigUint64Array ? type.Uint64 : + T extends Float32Array ? type.Float32 : + T extends Float64Array ? type.Float64 : + never; + +/** @ignore */ +type TypeToVector<T extends Type> = { + [key: number ]: vecs.Vector<any> ; + [Type.Null ]: vecs.NullVector ; + [Type.Bool ]: vecs.BoolVector ; + [Type.Int8 ]: vecs.Int8Vector ; + [Type.Int16 ]: vecs.Int16Vector ; + [Type.Int32 ]: vecs.Int32Vector ; + [Type.Int64 ]: vecs.Int64Vector ; + [Type.Uint8 ]: vecs.Uint8Vector ; + [Type.Uint16 ]: vecs.Uint16Vector ; + [Type.Uint32 ]: vecs.Uint32Vector ; + [Type.Uint64 ]: vecs.Uint64Vector ; + [Type.Int ]: vecs.IntVector ; + [Type.Float16 ]: vecs.Float16Vector ; + [Type.Float32 ]: vecs.Float32Vector ; + [Type.Float64 ]: vecs.Float64Vector ; + [Type.Float ]: vecs.FloatVector ; + [Type.Utf8 ]: vecs.Utf8Vector ; + [Type.Binary ]: vecs.BinaryVector ; + [Type.FixedSizeBinary ]: vecs.FixedSizeBinaryVector ; + [Type.Date ]: vecs.DateVector ; + [Type.DateDay ]: vecs.DateDayVector ; + [Type.DateMillisecond ]: vecs.DateMillisecondVector ; + [Type.Timestamp ]: vecs.TimestampVector ; + [Type.TimestampSecond ]: vecs.TimestampSecondVector ; + [Type.TimestampMillisecond ]: vecs.TimestampMillisecondVector ; + [Type.TimestampMicrosecond ]: vecs.TimestampMicrosecondVector ; + [Type.TimestampNanosecond ]: vecs.TimestampNanosecondVector ; + [Type.Time ]: vecs.TimeVector ; + [Type.TimeSecond ]: vecs.TimeSecondVector ; + [Type.TimeMillisecond ]: vecs.TimeMillisecondVector ; + [Type.TimeMicrosecond ]: vecs.TimeMicrosecondVector ; + [Type.TimeNanosecond ]: vecs.TimeNanosecondVector ; + [Type.Decimal ]: vecs.DecimalVector ; + [Type.Union ]: vecs.UnionVector ; + [Type.DenseUnion ]: vecs.DenseUnionVector ; + [Type.SparseUnion ]: vecs.SparseUnionVector ; + [Type.Interval ]: vecs.IntervalVector ; + [Type.IntervalDayTime ]: vecs.IntervalDayTimeVector ; + [Type.IntervalYearMonth ]: vecs.IntervalYearMonthVector ; + [Type.Map ]: vecs.MapVector ; + [Type.List ]: vecs.ListVector ; + [Type.Struct ]: vecs.StructVector ; + [Type.Dictionary ]: vecs.DictionaryVector ; + [Type.FixedSizeList ]: vecs.FixedSizeListVector ; +}[T]; + +/** @ignore */ +type DataTypeToVector<T extends DataType = any> = { + [key: number ]: vecs.Vector<any> ; + [Type.Null ]: T extends type.Null ? vecs.NullVector : never ; + [Type.Bool ]: T extends type.Bool ? vecs.BoolVector : never ; + [Type.Int8 ]: T extends type.Int8 ? vecs.Int8Vector : never ; + [Type.Int16 ]: T extends type.Int16 ? vecs.Int16Vector : never ; + [Type.Int32 ]: T extends type.Int32 ? vecs.Int32Vector : never ; + [Type.Int64 ]: T extends type.Int64 ? vecs.Int64Vector : never ; + [Type.Uint8 ]: T extends type.Uint8 ? vecs.Uint8Vector : never ; + [Type.Uint16 ]: T extends type.Uint16 ? vecs.Uint16Vector : never ; + [Type.Uint32 ]: T extends type.Uint32 ? vecs.Uint32Vector : never ; + [Type.Uint64 ]: T extends type.Uint64 ? vecs.Uint64Vector : never ; + [Type.Int ]: T extends type.Int ? vecs.IntVector : never ; + [Type.Float16 ]: T extends type.Float16 ? vecs.Float16Vector : never ; + [Type.Float32 ]: T extends type.Float32 ? vecs.Float32Vector : never ; + [Type.Float64 ]: T extends type.Float64 ? vecs.Float64Vector : never ; + [Type.Float ]: T extends type.Float ? vecs.FloatVector : never ; + [Type.Utf8 ]: T extends type.Utf8 ? vecs.Utf8Vector : never ; + [Type.Binary ]: T extends type.Binary ? vecs.BinaryVector : never ; + [Type.FixedSizeBinary ]: T extends type.FixedSizeBinary ? vecs.FixedSizeBinaryVector : never ; + [Type.Date ]: T extends type.Date_ ? vecs.DateVector : never ; + [Type.DateDay ]: T extends type.DateDay ? vecs.DateDayVector : never ; + [Type.DateMillisecond ]: T extends type.DateMillisecond ? vecs.DateMillisecondVector : never ; + [Type.Timestamp ]: T extends type.Timestamp ? vecs.TimestampVector : never ; + [Type.TimestampSecond ]: T extends type.TimestampSecond ? vecs.TimestampSecondVector : never ; + [Type.TimestampMillisecond ]: T extends type.TimestampMillisecond ? vecs.TimestampMillisecondVector : never ; + [Type.TimestampMicrosecond ]: T extends type.TimestampMicrosecond ? vecs.TimestampMicrosecondVector : never ; + [Type.TimestampNanosecond ]: T extends type.TimestampNanosecond ? vecs.TimestampNanosecondVector : never ; + [Type.Time ]: T extends type.Time ? vecs.TimeVector : never ; + [Type.TimeSecond ]: T extends type.TimeSecond ? vecs.TimeSecondVector : never ; + [Type.TimeMillisecond ]: T extends type.TimeMillisecond ? vecs.TimeMillisecondVector : never ; + [Type.TimeMicrosecond ]: T extends type.TimeMicrosecond ? vecs.TimeMicrosecondVector : never ; + [Type.TimeNanosecond ]: T extends type.TimeNanosecond ? vecs.TimeNanosecondVector : never ; + [Type.Decimal ]: T extends type.Decimal ? vecs.DecimalVector : never ; + [Type.Union ]: T extends type.Union ? vecs.UnionVector : never ; + [Type.DenseUnion ]: T extends type.DenseUnion ? vecs.DenseUnionVector : never ; + [Type.SparseUnion ]: T extends type.SparseUnion ? vecs.SparseUnionVector : never ; + [Type.Interval ]: T extends type.Interval ? vecs.IntervalVector : never ; + [Type.IntervalDayTime ]: T extends type.IntervalDayTime ? vecs.IntervalDayTimeVector : never ; + [Type.IntervalYearMonth ]: T extends type.IntervalYearMonth ? vecs.IntervalYearMonthVector : never ; + [Type.Map ]: T extends type.Map_ ? vecs.MapVector<T['keyType'], T['valueType']> : never ; + [Type.List ]: T extends type.List ? vecs.ListVector<T['valueType']> : never ; + [Type.Struct ]: T extends type.Struct ? vecs.StructVector<T['dataTypes']> : never ; + [Type.Dictionary ]: T extends type.Dictionary ? vecs.DictionaryVector<T['valueType'], T['indices']> : never ; + [Type.FixedSizeList ]: T extends type.FixedSizeList ? vecs.FixedSizeListVector<T['valueType']> : never ; +}[T['TType']]; + +/** @ignore */ +export type TypeToDataType<T extends Type> = { + [key: number ]: type.DataType ; + [Type.Null ]: type.Null ; + [Type.Bool ]: type.Bool ; + [Type.Int ]: type.Int ; + [Type.Int16 ]: type.Int16 ; + [Type.Int32 ]: type.Int32 ; + [Type.Int64 ]: type.Int64 ; + [Type.Uint8 ]: type.Uint8 ; + [Type.Uint16 ]: type.Uint16 ; + [Type.Uint32 ]: type.Uint32 ; + [Type.Uint64 ]: type.Uint64 ; + [Type.Int8 ]: type.Int8 ; + [Type.Float16 ]: type.Float16 ; + [Type.Float32 ]: type.Float32 ; + [Type.Float64 ]: type.Float64 ; + [Type.Float ]: type.Float ; + [Type.Utf8 ]: type.Utf8 ; + [Type.Binary ]: type.Binary ; + [Type.FixedSizeBinary ]: type.FixedSizeBinary ; + [Type.Date ]: type.Date_ ; + [Type.DateDay ]: type.DateDay ; + [Type.DateMillisecond ]: type.DateMillisecond ; + [Type.Timestamp ]: type.Timestamp ; + [Type.TimestampSecond ]: type.TimestampSecond ; + [Type.TimestampMillisecond ]: type.TimestampMillisecond ; + [Type.TimestampMicrosecond ]: type.TimestampMicrosecond ; + [Type.TimestampNanosecond ]: type.TimestampNanosecond ; + [Type.Time ]: type.Time ; + [Type.TimeSecond ]: type.TimeSecond ; + [Type.TimeMillisecond ]: type.TimeMillisecond ; + [Type.TimeMicrosecond ]: type.TimeMicrosecond ; + [Type.TimeNanosecond ]: type.TimeNanosecond ; + [Type.Decimal ]: type.Decimal ; + [Type.Union ]: type.Union ; + [Type.DenseUnion ]: type.DenseUnion ; + [Type.SparseUnion ]: type.SparseUnion ; + [Type.Interval ]: type.Interval ; + [Type.IntervalDayTime ]: type.IntervalDayTime ; + [Type.IntervalYearMonth ]: type.IntervalYearMonth ; + [Type.Map ]: type.Map_ ; + [Type.List ]: type.List ; + [Type.Struct ]: type.Struct ; + [Type.Dictionary ]: type.Dictionary ; + [Type.FixedSizeList ]: type.FixedSizeList ; +}[T]; + +/** @ignore */ +type TypeToBuilder<T extends Type = any, TNull = any> = { + [key: number ]: builders.Builder ; + [Type.Null ]: builders.NullBuilder<TNull> ; + [Type.Bool ]: builders.BoolBuilder<TNull> ; + [Type.Int8 ]: builders.Int8Builder<TNull> ; + [Type.Int16 ]: builders.Int16Builder<TNull> ; + [Type.Int32 ]: builders.Int32Builder<TNull> ; + [Type.Int64 ]: builders.Int64Builder<TNull> ; + [Type.Uint8 ]: builders.Uint8Builder<TNull> ; + [Type.Uint16 ]: builders.Uint16Builder<TNull> ; + [Type.Uint32 ]: builders.Uint32Builder<TNull> ; + [Type.Uint64 ]: builders.Uint64Builder<TNull> ; + [Type.Int ]: builders.IntBuilder<any, TNull> ; + [Type.Float16 ]: builders.Float16Builder<TNull> ; + [Type.Float32 ]: builders.Float32Builder<TNull> ; + [Type.Float64 ]: builders.Float64Builder<TNull> ; + [Type.Float ]: builders.FloatBuilder<any, TNull> ; + [Type.Utf8 ]: builders.Utf8Builder<TNull> ; + [Type.Binary ]: builders.BinaryBuilder<TNull> ; + [Type.FixedSizeBinary ]: builders.FixedSizeBinaryBuilder<TNull> ; + [Type.Date ]: builders.DateBuilder<any, TNull> ; + [Type.DateDay ]: builders.DateDayBuilder<TNull> ; + [Type.DateMillisecond ]: builders.DateMillisecondBuilder<TNull> ; + [Type.Timestamp ]: builders.TimestampBuilder<any, TNull> ; + [Type.TimestampSecond ]: builders.TimestampSecondBuilder<TNull> ; + [Type.TimestampMillisecond ]: builders.TimestampMillisecondBuilder<TNull> ; + [Type.TimestampMicrosecond ]: builders.TimestampMicrosecondBuilder<TNull> ; + [Type.TimestampNanosecond ]: builders.TimestampNanosecondBuilder<TNull> ; + [Type.Time ]: builders.TimeBuilder<any, TNull> ; + [Type.TimeSecond ]: builders.TimeSecondBuilder<TNull> ; + [Type.TimeMillisecond ]: builders.TimeMillisecondBuilder<TNull> ; + [Type.TimeMicrosecond ]: builders.TimeMicrosecondBuilder<TNull> ; + [Type.TimeNanosecond ]: builders.TimeNanosecondBuilder<TNull> ; + [Type.Decimal ]: builders.DecimalBuilder<TNull> ; + [Type.Union ]: builders.UnionBuilder<any, TNull> ; + [Type.DenseUnion ]: builders.DenseUnionBuilder<any, TNull> ; + [Type.SparseUnion ]: builders.SparseUnionBuilder<any, TNull> ; + [Type.Interval ]: builders.IntervalBuilder<any, TNull> ; + [Type.IntervalDayTime ]: builders.IntervalDayTimeBuilder<TNull> ; + [Type.IntervalYearMonth ]: builders.IntervalYearMonthBuilder<TNull> ; + [Type.Map ]: builders.MapBuilder<any, any, TNull> ; + [Type.List ]: builders.ListBuilder<any, TNull> ; + [Type.Struct ]: builders.StructBuilder<any, TNull> ; + [Type.Dictionary ]: builders.DictionaryBuilder<any, TNull> ; + [Type.FixedSizeList ]: builders.FixedSizeListBuilder<any, TNull> ; +}[T]; + +/** @ignore */ +type DataTypeToBuilder<T extends DataType = any, TNull = any> = { + [key: number ]: builders.Builder<any, TNull> ; + [Type.Null ]: T extends type.Null ? builders.NullBuilder<TNull> : never ; + [Type.Bool ]: T extends type.Bool ? builders.BoolBuilder<TNull> : never ; + [Type.Int8 ]: T extends type.Int8 ? builders.Int8Builder<TNull> : never ; + [Type.Int16 ]: T extends type.Int16 ? builders.Int16Builder<TNull> : never ; + [Type.Int32 ]: T extends type.Int32 ? builders.Int32Builder<TNull> : never ; + [Type.Int64 ]: T extends type.Int64 ? builders.Int64Builder<TNull> : never ; + [Type.Uint8 ]: T extends type.Uint8 ? builders.Uint8Builder<TNull> : never ; + [Type.Uint16 ]: T extends type.Uint16 ? builders.Uint16Builder<TNull> : never ; + [Type.Uint32 ]: T extends type.Uint32 ? builders.Uint32Builder<TNull> : never ; + [Type.Uint64 ]: T extends type.Uint64 ? builders.Uint64Builder<TNull> : never ; + [Type.Int ]: T extends type.Int ? builders.IntBuilder<T, TNull> : never ; + [Type.Float16 ]: T extends type.Float16 ? builders.Float16Builder<TNull> : never ; + [Type.Float32 ]: T extends type.Float32 ? builders.Float32Builder<TNull> : never ; + [Type.Float64 ]: T extends type.Float64 ? builders.Float64Builder<TNull> : never ; + [Type.Float ]: T extends type.Float ? builders.FloatBuilder<T, TNull> : never ; + [Type.Utf8 ]: T extends type.Utf8 ? builders.Utf8Builder<TNull> : never ; + [Type.Binary ]: T extends type.Binary ? builders.BinaryBuilder<TNull> : never ; + [Type.FixedSizeBinary ]: T extends type.FixedSizeBinary ? builders.FixedSizeBinaryBuilder<TNull> : never ; + [Type.Date ]: T extends type.Date_ ? builders.DateBuilder<T, TNull> : never ; + [Type.DateDay ]: T extends type.DateDay ? builders.DateDayBuilder<TNull> : never ; + [Type.DateMillisecond ]: T extends type.DateMillisecond ? builders.DateMillisecondBuilder<TNull> : never ; + [Type.Timestamp ]: T extends type.Timestamp ? builders.TimestampBuilder<T, TNull> : never ; + [Type.TimestampSecond ]: T extends type.TimestampSecond ? builders.TimestampSecondBuilder<TNull> : never ; + [Type.TimestampMillisecond ]: T extends type.TimestampMillisecond ? builders.TimestampMillisecondBuilder<TNull> : never ; + [Type.TimestampMicrosecond ]: T extends type.TimestampMicrosecond ? builders.TimestampMicrosecondBuilder<TNull> : never ; + [Type.TimestampNanosecond ]: T extends type.TimestampNanosecond ? builders.TimestampNanosecondBuilder<TNull> : never ; + [Type.Time ]: T extends type.Time ? builders.TimeBuilder<T, TNull> : never ; + [Type.TimeSecond ]: T extends type.TimeSecond ? builders.TimeSecondBuilder<TNull> : never ; + [Type.TimeMillisecond ]: T extends type.TimeMillisecond ? builders.TimeMillisecondBuilder<TNull> : never ; + [Type.TimeMicrosecond ]: T extends type.TimeMicrosecond ? builders.TimeMicrosecondBuilder<TNull> : never ; + [Type.TimeNanosecond ]: T extends type.TimeNanosecond ? builders.TimeNanosecondBuilder<TNull> : never ; + [Type.Decimal ]: T extends type.Decimal ? builders.DecimalBuilder<TNull> : never ; + [Type.Union ]: T extends type.Union ? builders.UnionBuilder<T, TNull> : never ; + [Type.DenseUnion ]: T extends type.DenseUnion ? builders.DenseUnionBuilder<T, TNull> : never ; + [Type.SparseUnion ]: T extends type.SparseUnion ? builders.SparseUnionBuilder<T, TNull> : never ; + [Type.Interval ]: T extends type.Interval ? builders.IntervalBuilder<T, TNull> : never ; + [Type.IntervalDayTime ]: T extends type.IntervalDayTime ? builders.IntervalDayTimeBuilder<TNull> : never ; + [Type.IntervalYearMonth ]: T extends type.IntervalYearMonth ? builders.IntervalYearMonthBuilder<TNull> : never ; + [Type.Map ]: T extends type.Map_ ? builders.MapBuilder<T['keyType'], T['valueType'], TNull> : never ; + [Type.List ]: T extends type.List ? builders.ListBuilder<T['valueType'], TNull> : never ; + [Type.Struct ]: T extends type.Struct ? builders.StructBuilder<T['dataTypes'], TNull> : never ; + [Type.Dictionary ]: T extends type.Dictionary ? builders.DictionaryBuilder<T, TNull> : never ; + [Type.FixedSizeList ]: T extends type.FixedSizeList ? builders.FixedSizeListBuilder<T['valueType'], TNull> : never ; +}[T['TType']]; diff --git a/src/arrow/js/src/io/adapters.ts b/src/arrow/js/src/io/adapters.ts new file mode 100644 index 000000000..a83346ef7 --- /dev/null +++ b/src/arrow/js/src/io/adapters.ts @@ -0,0 +1,398 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { + toUint8Array, + joinUint8Arrays, + ArrayBufferViewInput, + toUint8ArrayIterator, + toUint8ArrayAsyncIterator +} from '../util/buffer'; + +import { ReadableDOMStreamOptions } from './interfaces'; + +interface ReadableStreamReadResult<T> { done: boolean; value: T } +type Uint8ArrayGenerator = Generator<Uint8Array, null, { cmd: 'peek' | 'read'; size: number }>; +type AsyncUint8ArrayGenerator = AsyncGenerator<Uint8Array, null, { cmd: 'peek' | 'read'; size: number }>; + +/** @ignore */ +export default { + fromIterable<T extends ArrayBufferViewInput>(source: Iterable<T> | T): Uint8ArrayGenerator { + return pump(fromIterable<T>(source)); + }, + fromAsyncIterable<T extends ArrayBufferViewInput>(source: AsyncIterable<T> | PromiseLike<T>): AsyncUint8ArrayGenerator { + return pump(fromAsyncIterable<T>(source)); + }, + fromDOMStream<T extends ArrayBufferViewInput>(source: ReadableStream<T>): AsyncUint8ArrayGenerator { + return pump(fromDOMStream<T>(source)); + }, + fromNodeStream(stream: NodeJS.ReadableStream): AsyncUint8ArrayGenerator { + return pump(fromNodeStream(stream)); + }, + // @ts-ignore + toDOMStream<T>(source: Iterable<T> | AsyncIterable<T>, options?: ReadableDOMStreamOptions): ReadableStream<T> { + throw new Error(`"toDOMStream" not available in this environment`); + }, + // @ts-ignore + toNodeStream<T>(source: Iterable<T> | AsyncIterable<T>, options?: import('stream').ReadableOptions): import('stream').Readable { + throw new Error(`"toNodeStream" not available in this environment`); + }, +}; + +/** @ignore */ +const pump = <T extends Uint8ArrayGenerator | AsyncUint8ArrayGenerator>(iterator: T) => { iterator.next(); return iterator; }; + +/** @ignore */ +function* fromIterable<T extends ArrayBufferViewInput>(source: Iterable<T> | T): Uint8ArrayGenerator { + + let done: boolean | undefined, threw = false; + let buffers: Uint8Array[] = [], buffer: Uint8Array; + let cmd: 'peek' | 'read', size: number, bufferLength = 0; + + function byteRange() { + if (cmd === 'peek') { + return joinUint8Arrays(buffers, size)[0]; + } + [buffer, buffers, bufferLength] = joinUint8Arrays(buffers, size); + return buffer; + } + + // Yield so the caller can inject the read command before creating the source Iterator + ({ cmd, size } = yield <any> null); + + // initialize the iterator + const it = toUint8ArrayIterator(source)[Symbol.iterator](); + + try { + do { + // read the next value + ({ done, value: buffer } = isNaN(size - bufferLength) ? + it.next(undefined) : it.next(size - bufferLength)); + // if chunk is not null or empty, push it onto the queue + if (!done && buffer.byteLength > 0) { + buffers.push(buffer); + bufferLength += buffer.byteLength; + } + // If we have enough bytes in our buffer, yield chunks until we don't + if (done || size <= bufferLength) { + do { + ({ cmd, size } = yield byteRange()); + } while (size < bufferLength); + } + } while (!done); + } catch (e) { + (threw = true) && (typeof it.throw === 'function') && (it.throw(e)); + } finally { + (threw === false) && (typeof it.return === 'function') && (it.return(null!)); + } + return null; +} + +/** @ignore */ +async function* fromAsyncIterable<T extends ArrayBufferViewInput>(source: AsyncIterable<T> | PromiseLike<T>): AsyncUint8ArrayGenerator { + + let done: boolean | undefined, threw = false; + let buffers: Uint8Array[] = [], buffer: Uint8Array; + let cmd: 'peek' | 'read', size: number, bufferLength = 0; + + function byteRange() { + if (cmd === 'peek') { + return joinUint8Arrays(buffers, size)[0]; + } + [buffer, buffers, bufferLength] = joinUint8Arrays(buffers, size); + return buffer; + } + + // Yield so the caller can inject the read command before creating the source AsyncIterator + ({ cmd, size } = (yield <any> null)!); + + // initialize the iterator + const it = toUint8ArrayAsyncIterator(source)[Symbol.asyncIterator](); + + try { + do { + // read the next value + ({ done, value: buffer } = isNaN(size - bufferLength) + ? await it.next(undefined) + : await it.next(size - bufferLength)); + // if chunk is not null or empty, push it onto the queue + if (!done && buffer.byteLength > 0) { + buffers.push(buffer); + bufferLength += buffer.byteLength; + } + // If we have enough bytes in our buffer, yield chunks until we don't + if (done || size <= bufferLength) { + do { + ({ cmd, size } = yield byteRange()); + } while (size < bufferLength); + } + } while (!done); + } catch (e) { + (threw = true) && (typeof it.throw === 'function') && (await it.throw(e)); + } finally { + (threw === false) && (typeof it.return === 'function') && (await it.return(new Uint8Array(0))); + } + return null; +} + +// All this manual Uint8Array chunk management can be avoided if/when engines +// add support for ArrayBuffer.transfer() or ArrayBuffer.prototype.realloc(): +// https://github.com/domenic/proposal-arraybuffer-transfer +/** @ignore */ +async function* fromDOMStream<T extends ArrayBufferViewInput>(source: ReadableStream<T>): AsyncUint8ArrayGenerator { + + let done = false, threw = false; + let buffers: Uint8Array[] = [], buffer: Uint8Array; + let cmd: 'peek' | 'read', size: number, bufferLength = 0; + + function byteRange() { + if (cmd === 'peek') { + return joinUint8Arrays(buffers, size)[0]; + } + [buffer, buffers, bufferLength] = joinUint8Arrays(buffers, size); + return buffer; + } + + // Yield so the caller can inject the read command before we establish the ReadableStream lock + ({ cmd, size } = yield <any> null); + + // initialize the reader and lock the stream + const it = new AdaptiveByteReader(source); + + try { + do { + // read the next value + ({ done, value: buffer } = isNaN(size - bufferLength) + ? await it['read'](undefined) + : await it['read'](size - bufferLength)); + // if chunk is not null or empty, push it onto the queue + if (!done && buffer.byteLength > 0) { + buffers.push(toUint8Array(buffer)); + bufferLength += buffer.byteLength; + } + // If we have enough bytes in our buffer, yield chunks until we don't + if (done || size <= bufferLength) { + do { + ({ cmd, size } = yield byteRange()); + } while (size < bufferLength); + } + } while (!done); + } catch (e) { + (threw = true) && (await it['cancel'](e)); + } finally { + (threw === false) ? (await it['cancel']()) + : source['locked'] && it.releaseLock(); + } + return null; +} + +/** @ignore */ +class AdaptiveByteReader<T extends ArrayBufferViewInput> { + + private supportsBYOB: boolean; + private byobReader: ReadableStreamBYOBReader | null = null; + private defaultReader: ReadableStreamDefaultReader<T> | null = null; + private reader: ReadableStreamBYOBReader | ReadableStreamDefaultReader<T> | null; + + constructor(private source: ReadableStream<T>) { + try { + this.supportsBYOB = !!(this.reader = this.getBYOBReader()); + } catch (e) { + this.supportsBYOB = !(this.reader = this.getDefaultReader()); + } + } + + get closed(): Promise<void> { + return this.reader ? this.reader['closed'].catch(() => {}) : Promise.resolve(); + } + + releaseLock(): void { + if (this.reader) { + this.reader.releaseLock(); + } + this.reader = this.byobReader = this.defaultReader = null; + } + + async cancel(reason?: any): Promise<void> { + const { reader, source } = this; + reader && (await reader['cancel'](reason).catch(() => {})); + source && (source['locked'] && this.releaseLock()); + } + + async read(size?: number): Promise<ReadableStreamReadResult<Uint8Array>> { + if (size === 0) { + return { done: this.reader == null, value: new Uint8Array(0) }; + } + const result = !this.supportsBYOB || typeof size !== 'number' + ? await this.getDefaultReader().read() + : await this.readFromBYOBReader(size); + !result.done && (result.value = toUint8Array(result as ReadableStreamReadResult<Uint8Array>)); + return result as ReadableStreamReadResult<Uint8Array>; + } + + private getDefaultReader() { + if (this.byobReader) { this.releaseLock(); } + if (!this.defaultReader) { + this.defaultReader = this.source['getReader'](); + // We have to catch and swallow errors here to avoid uncaught promise rejection exceptions + // that seem to be raised when we call `releaseLock()` on this reader. I'm still mystified + // about why these errors are raised, but I'm sure there's some important spec reason that + // I haven't considered. I hate to employ such an anti-pattern here, but it seems like the + // only solution in this case :/ + this.defaultReader['closed'].catch(() => {}); + } + return (this.reader = this.defaultReader); + } + + private getBYOBReader() { + if (this.defaultReader) { this.releaseLock(); } + if (!this.byobReader) { + this.byobReader = this.source['getReader']({ mode: 'byob' }); + // We have to catch and swallow errors here to avoid uncaught promise rejection exceptions + // that seem to be raised when we call `releaseLock()` on this reader. I'm still mystified + // about why these errors are raised, but I'm sure there's some important spec reason that + // I haven't considered. I hate to employ such an anti-pattern here, but it seems like the + // only solution in this case :/ + this.byobReader['closed'].catch(() => {}); + } + return (this.reader = this.byobReader); + } + + // This strategy plucked from the example in the streams spec: + // https://streams.spec.whatwg.org/#example-manual-read-bytes + private async readFromBYOBReader(size: number) { + return await readInto(this.getBYOBReader(), new ArrayBuffer(size), 0, size); + } +} + +/** @ignore */ +async function readInto(reader: ReadableStreamBYOBReader, buffer: ArrayBufferLike, offset: number, size: number): Promise<ReadableStreamReadResult<Uint8Array>> { + if (offset >= size) { + return { done: false, value: new Uint8Array(buffer, 0, size) }; + } + const { done, value } = await reader.read(new Uint8Array(buffer, offset, size - offset)); + if (((offset += value!.byteLength) < size) && !done) { + return await readInto(reader, value!.buffer, offset, size); + } + return { done, value: new Uint8Array(value!.buffer, 0, offset) }; +} + +/** @ignore */ +type EventName = 'end' | 'error' | 'readable'; +/** @ignore */ +type Event = [EventName, (_: any) => void, Promise<[EventName, Error | null]>]; +/** @ignore */ +const onEvent = <T extends string>(stream: NodeJS.ReadableStream, event: T) => { + const handler = (_: any) => resolve([event, _]); + let resolve: (value?: [T, any] | PromiseLike<[T, any]>) => void; + return [event, handler, new Promise<[T, any]>( + (r) => (resolve = r) && stream['once'](event, handler) + )] as Event; +}; + +/** @ignore */ +async function* fromNodeStream(stream: NodeJS.ReadableStream): AsyncUint8ArrayGenerator { + + const events: Event[] = []; + let event: EventName = 'error'; + let done = false, err: Error | null = null; + let cmd: 'peek' | 'read', size: number, bufferLength = 0; + let buffers: Uint8Array[] = [], buffer: Uint8Array | Buffer | string; + + function byteRange() { + if (cmd === 'peek') { + return joinUint8Arrays(buffers, size)[0]; + } + [buffer, buffers, bufferLength] = joinUint8Arrays(buffers, size); + return buffer; + } + + // Yield so the caller can inject the read command before we + // add the listener for the source stream's 'readable' event. + ({ cmd, size } = yield <any> null); + + // ignore stdin if it's a TTY + if ((stream as any)['isTTY']) { + yield new Uint8Array(0); + return null; + } + + try { + // initialize the stream event handlers + events[0] = onEvent(stream, 'end'); + events[1] = onEvent(stream, 'error'); + + do { + events[2] = onEvent(stream, 'readable'); + + // wait on the first message event from the stream + [event, err] = await Promise.race(events.map((x) => x[2])); + + // if the stream emitted an Error, rethrow it + if (event === 'error') { break; } + if (!(done = event === 'end')) { + // If the size is NaN, request to read everything in the stream's internal buffer + if (!isFinite(size - bufferLength)) { + buffer = toUint8Array(stream['read'](undefined)); + } else { + buffer = toUint8Array(stream['read'](size - bufferLength)); + // If the byteLength is 0, then the requested amount is more than the stream has + // in its internal buffer. In this case the stream needs a "kick" to tell it to + // continue emitting readable events, so request to read everything the stream + // has in its internal buffer right now. + if (buffer.byteLength < (size - bufferLength)) { + buffer = toUint8Array(stream['read'](undefined)); + } + } + // if chunk is not null or empty, push it onto the queue + if (buffer.byteLength > 0) { + buffers.push(buffer); + bufferLength += buffer.byteLength; + } + } + // If we have enough bytes in our buffer, yield chunks until we don't + if (done || size <= bufferLength) { + do { + ({ cmd, size } = yield byteRange()); + } while (size < bufferLength); + } + } while (!done); + } finally { + await cleanup(events, event === 'error' ? err : null); + } + + return null; + + function cleanup<T extends Error | null | void>(events: Event[], err?: T) { + buffer = buffers = <any> null; + return new Promise<T>((resolve, reject) => { + for (const [evt, fn] of events) { + stream['off'](evt, fn); + } + try { + // Some stream implementations don't call the destroy callback, + // because it's really a node-internal API. Just calling `destroy` + // here should be enough to conform to the ReadableStream contract + const destroy = (stream as any)['destroy']; + destroy && destroy.call(stream, err); + err = undefined; + } catch (e) { err = e || err; } finally { + err != null ? reject(err) : resolve(); + } + }); + } +} diff --git a/src/arrow/js/src/io/file.ts b/src/arrow/js/src/io/file.ts new file mode 100644 index 000000000..20b7dbf02 --- /dev/null +++ b/src/arrow/js/src/io/file.ts @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { FileHandle } from './interfaces'; +import { ByteStream, AsyncByteStream } from './stream'; +import { ArrayBufferViewInput, toUint8Array } from '../util/buffer'; + +/** @ignore */ +export class RandomAccessFile extends ByteStream { + public size: number; + public position = 0; + protected buffer: Uint8Array | null; + constructor(buffer: ArrayBufferViewInput, byteLength?: number) { + super(); + this.buffer = toUint8Array(buffer); + this.size = typeof byteLength === 'undefined' ? this.buffer.byteLength : byteLength; + } + public readInt32(position: number) { + const { buffer, byteOffset } = this.readAt(position, 4); + return new DataView(buffer, byteOffset).getInt32(0, true); + } + public seek(position: number) { + this.position = Math.min(position, this.size); + return position < this.size; + } + public read(nBytes?: number | null) { + const { buffer, size, position } = this; + if (buffer && position < size) { + if (typeof nBytes !== 'number') { nBytes = Infinity; } + this.position = Math.min(size, + position + Math.min(size - position, nBytes)); + return buffer.subarray(position, this.position); + } + return null; + } + public readAt(position: number, nBytes: number) { + const buf = this.buffer; + const end = Math.min(this.size, position + nBytes); + return buf ? buf.subarray(position, end) : new Uint8Array(nBytes); + } + public close() { this.buffer && (this.buffer = null); } + public throw(value?: any) { this.close(); return { done: true, value }; } + public return(value?: any) { this.close(); return { done: true, value }; } +} + +/** @ignore */ +export class AsyncRandomAccessFile extends AsyncByteStream { + public size!: number; + public position = 0; + public _pending?: Promise<void>; + protected _handle: FileHandle | null; + constructor(file: FileHandle, byteLength?: number) { + super(); + this._handle = file; + if (typeof byteLength === 'number') { + this.size = byteLength; + } else { + this._pending = (async () => { + this.size = (await file.stat()).size; + delete this._pending; + })(); + } + } + public async readInt32(position: number) { + const { buffer, byteOffset } = await this.readAt(position, 4); + return new DataView(buffer, byteOffset).getInt32(0, true); + } + public async seek(position: number) { + this._pending && await this._pending; + this.position = Math.min(position, this.size); + return position < this.size; + } + public async read(nBytes?: number | null) { + this._pending && await this._pending; + const { _handle: file, size, position } = this; + if (file && position < size) { + if (typeof nBytes !== 'number') { nBytes = Infinity; } + let pos = position, offset = 0, bytesRead = 0; + const end = Math.min(size, pos + Math.min(size - pos, nBytes)); + const buffer = new Uint8Array(Math.max(0, (this.position = end) - pos)); + while ((pos += bytesRead) < end && (offset += bytesRead) < buffer.byteLength) { + ({ bytesRead } = await file.read(buffer, offset, buffer.byteLength - offset, pos)); + } + return buffer; + } + return null; + } + public async readAt(position: number, nBytes: number) { + this._pending && await this._pending; + const { _handle: file, size } = this; + if (file && (position + nBytes) < size) { + const end = Math.min(size, position + nBytes); + const buffer = new Uint8Array(end - position); + return (await file.read(buffer, 0, nBytes, position)).buffer; + } + return new Uint8Array(nBytes); + } + public async close() { const f = this._handle; this._handle = null; f && await f.close(); } + public async throw(value?: any) { await this.close(); return { done: true, value }; } + public async return(value?: any) { await this.close(); return { done: true, value }; } +} diff --git a/src/arrow/js/src/io/interfaces.ts b/src/arrow/js/src/io/interfaces.ts new file mode 100644 index 000000000..4b5641ff1 --- /dev/null +++ b/src/arrow/js/src/io/interfaces.ts @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import streamAdapters from './adapters'; + +/** @ignore */ +export const ITERATOR_DONE: any = Object.freeze({ done: true, value: void (0) }); + +/** @ignore */ +export type FileHandle = import('fs').promises.FileHandle; +/** @ignore */ +export type ArrowJSONLike = { schema: any; batches?: any[]; dictionaries?: any[] }; +/** @ignore */ +export type ReadableDOMStreamOptions = { type: 'bytes' | undefined; autoAllocateChunkSize?: number; highWaterMark?: number }; + +/** @ignore */ +export class ArrowJSON { + constructor(private _json: ArrowJSONLike) {} + public get schema(): any { return this._json['schema']; } + public get batches(): any[] { return (this._json['batches'] || []) as any[]; } + public get dictionaries(): any[] { return (this._json['dictionaries'] || []) as any[]; } +} + +/** @ignore */ +export interface Readable<T> { + + readonly closed: Promise<void>; + cancel(reason?: any): Promise<void>; + + read(size?: number | null): Promise<T | null>; + peek(size?: number | null): Promise<T | null>; + throw(value?: any): Promise<IteratorResult<any>>; + return(value?: any): Promise<IteratorResult<any>>; + next(size?: number | null): Promise<IteratorResult<T>>; +} + +/** @ignore */ +export interface Writable<T> { + readonly closed: Promise<void>; + close(): void; + write(chunk: T): void; + abort(reason?: any): void; +} + +/** @ignore */ +export interface ReadableWritable<TReadable, TWritable> extends Readable<TReadable>, Writable<TWritable> { + [Symbol.asyncIterator](): AsyncIterableIterator<TReadable>; + toDOMStream(options?: ReadableDOMStreamOptions): ReadableStream<TReadable>; + toNodeStream(options?: import('stream').ReadableOptions): import('stream').Readable; +} + +/** @ignore */ +export abstract class ReadableInterop<T> { + + public abstract toDOMStream(options?: ReadableDOMStreamOptions): ReadableStream<T>; + public abstract toNodeStream(options?: import('stream').ReadableOptions): import('stream').Readable; + + public tee(): [ReadableStream<T>, ReadableStream<T>] { + return this._getDOMStream().tee(); + } + public pipe<R extends NodeJS.WritableStream>(writable: R, options?: { end?: boolean }) { + return this._getNodeStream().pipe(writable, options); + } + public pipeTo(writable: WritableStream<T>, options?: PipeOptions) { return this._getDOMStream().pipeTo(writable, options); } + public pipeThrough<R extends ReadableStream<any>>(duplex: { writable: WritableStream<T>; readable: R }, options?: PipeOptions) { + return this._getDOMStream().pipeThrough(duplex, options); + } + + protected _DOMStream?: ReadableStream<T>; + private _getDOMStream() { + return this._DOMStream || (this._DOMStream = this.toDOMStream()); + } + + protected _nodeStream?: import('stream').Readable; + private _getNodeStream() { + return this._nodeStream || (this._nodeStream = this.toNodeStream()); + } +} + +/** @ignore */ +type Resolution<T> = { resolve: (value?: T | PromiseLike<T>) => void; reject: (reason?: any) => void }; + +/** @ignore */ +export class AsyncQueue<TReadable = Uint8Array, TWritable = TReadable> extends ReadableInterop<TReadable> + implements AsyncIterableIterator<TReadable>, ReadableWritable<TReadable, TWritable> { + + protected _values: TWritable[] = []; + protected _error?: { error: any }; + protected _closedPromise: Promise<void>; + protected _closedPromiseResolve?: (value?: any) => void; + protected resolvers: Resolution<IteratorResult<TReadable>>[] = []; + + constructor() { + super(); + this._closedPromise = new Promise((r) => this._closedPromiseResolve = r); + } + + public get closed(): Promise<void> { return this._closedPromise; } + public async cancel(reason?: any) { await this.return(reason); } + public write(value: TWritable) { + if (this._ensureOpen()) { + this.resolvers.length <= 0 + ? (this._values.push(value)) + : (this.resolvers.shift()!.resolve({ done: false, value } as any)); + } + } + public abort(value?: any) { + if (this._closedPromiseResolve) { + this.resolvers.length <= 0 + ? (this._error = { error: value }) + : (this.resolvers.shift()!.reject({ done: true, value })); + } + } + public close() { + if (this._closedPromiseResolve) { + const { resolvers } = this; + while (resolvers.length > 0) { + resolvers.shift()!.resolve(ITERATOR_DONE); + } + this._closedPromiseResolve(); + this._closedPromiseResolve = undefined; + } + } + + public [Symbol.asyncIterator]() { return this; } + public toDOMStream(options?: ReadableDOMStreamOptions) { + return streamAdapters.toDOMStream( + (this._closedPromiseResolve || this._error) + ? (this as AsyncIterable<TReadable>) + : (this._values as any) as Iterable<TReadable>, + options); + } + public toNodeStream(options?: import('stream').ReadableOptions) { + return streamAdapters.toNodeStream( + (this._closedPromiseResolve || this._error) + ? (this as AsyncIterable<TReadable>) + : (this._values as any) as Iterable<TReadable>, + options); + } + public async throw(_?: any) { await this.abort(_); return ITERATOR_DONE; } + public async return(_?: any) { await this.close(); return ITERATOR_DONE; } + + public async read(size?: number | null): Promise<TReadable | null> { return (await this.next(size, 'read')).value; } + public async peek(size?: number | null): Promise<TReadable | null> { return (await this.next(size, 'peek')).value; } + public next(..._args: any[]): Promise<IteratorResult<TReadable>> { + if (this._values.length > 0) { + return Promise.resolve({ done: false, value: this._values.shift()! } as any); + } else if (this._error) { + return Promise.reject({ done: true, value: this._error.error }); + } else if (!this._closedPromiseResolve) { + return Promise.resolve(ITERATOR_DONE); + } else { + return new Promise<IteratorResult<TReadable>>((resolve, reject) => { + this.resolvers.push({ resolve, reject }); + }); + } + } + + protected _ensureOpen() { + if (this._closedPromiseResolve) { + return true; + } + throw new Error(`AsyncQueue is closed`); + } +} diff --git a/src/arrow/js/src/io/node/builder.ts b/src/arrow/js/src/io/node/builder.ts new file mode 100644 index 000000000..eb9579536 --- /dev/null +++ b/src/arrow/js/src/io/node/builder.ts @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Duplex } from 'stream'; +import { DataType } from '../../type'; +import { Builder, BuilderOptions } from '../../builder/index'; + +/** @ignore */ +export interface BuilderDuplexOptions<T extends DataType = any, TNull = any> extends BuilderOptions<T, TNull> { + autoDestroy?: boolean; + highWaterMark?: number; + queueingStrategy?: 'bytes' | 'count'; + dictionaryHashFunction?: (value: any) => string | number; + valueToChildTypeId?: (builder: Builder<T, TNull>, value: any, offset: number) => number; +} + +/** @ignore */ +export function builderThroughNodeStream<T extends DataType = any, TNull = any>(options: BuilderDuplexOptions<T, TNull>) { + return new BuilderDuplex(Builder.new(options), options); +} + +/** @ignore */ +type CB = (error?: Error | null | undefined) => void; + +/** @ignore */ +class BuilderDuplex<T extends DataType = any, TNull = any> extends Duplex { + + private _finished: boolean; + private _numChunks: number; + private _desiredSize: number; + private _builder: Builder<T, TNull>; + private _getSize: (builder: Builder<T, TNull>) => number; + + constructor(builder: Builder<T, TNull>, options: BuilderDuplexOptions<T, TNull>) { + + const { queueingStrategy = 'count', autoDestroy = true } = options; + const { highWaterMark = queueingStrategy !== 'bytes' ? 1000 : 2 ** 14 } = options; + + super({ autoDestroy, highWaterMark: 1, allowHalfOpen: true, writableObjectMode: true, readableObjectMode: true }); + + this._numChunks = 0; + this._finished = false; + this._builder = builder; + this._desiredSize = highWaterMark; + this._getSize = queueingStrategy !== 'bytes' ? builderLength : builderByteLength; + } + _read(size: number) { + this._maybeFlush(this._builder, this._desiredSize = size); + } + _final(cb?: CB) { + this._maybeFlush(this._builder.finish(), this._desiredSize); + cb && cb(); + } + _write(value: any, _: string, cb?: CB) { + const result = this._maybeFlush( + this._builder.append(value), + this._desiredSize + ); + cb && cb(); + return result; + } + _destroy(err: Error | null, cb?: (error: Error | null) => void) { + this._builder.clear(); + cb && cb(err); + } + private _maybeFlush(builder: Builder<T, TNull>, size: number) { + if (this._getSize(builder) >= size) { + ++this._numChunks && this.push(builder.toVector()); + } + if (builder.finished) { + if (builder.length > 0 || this._numChunks === 0) { + ++this._numChunks && this.push(builder.toVector()); + } + if (!this._finished && (this._finished = true)) { + this.push(null); + } + return false; + } + return this._getSize(builder) < this.writableHighWaterMark; + } +} + +/** @ignore */ const builderLength = <T extends DataType = any>(builder: Builder<T>) => builder.length; +/** @ignore */ const builderByteLength = <T extends DataType = any>(builder: Builder<T>) => builder.byteLength; diff --git a/src/arrow/js/src/io/node/iterable.ts b/src/arrow/js/src/io/node/iterable.ts new file mode 100644 index 000000000..457bc894d --- /dev/null +++ b/src/arrow/js/src/io/node/iterable.ts @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Readable } from 'stream'; +import { isIterable, isAsyncIterable } from '../../util/compat'; + +/** @ignore */ +type ReadableOptions = import('stream').ReadableOptions; +/** @ignore */ +type SourceIterator<T> = Generator<T, void, number | null>; +/** @ignore */ +type AsyncSourceIterator<T> = AsyncGenerator<T, void, number | null>; + +/** @ignore */ +export function toNodeStream<T>(source: Iterable<T> | AsyncIterable<T>, options?: ReadableOptions): Readable { + if (isAsyncIterable<T>(source)) { return new AsyncIterableReadable(source[Symbol.asyncIterator]() as AsyncSourceIterator<T>, options); } + if (isIterable<T>(source)) { return new IterableReadable(source[Symbol.iterator]() as SourceIterator<T>, options); } + /* istanbul ignore next */ + throw new Error(`toNodeStream() must be called with an Iterable or AsyncIterable`); +} + +/** @ignore */ +class IterableReadable<T extends Uint8Array | any> extends Readable { + private _pulling: boolean; + private _bytesMode: boolean; + private _iterator: SourceIterator<T>; + constructor(it: SourceIterator<T>, options?: ReadableOptions) { + super(options); + this._iterator = it; + this._pulling = false; + this._bytesMode = !options || !options.objectMode; + } + _read(size: number) { + const it = this._iterator; + if (it && !this._pulling && (this._pulling = true)) { + this._pulling = this._pull(size, it); + } + } + _destroy(e: Error | null, cb: (e: Error | null) => void) { + const it = this._iterator; + let fn: any; + it && (fn = e != null && it.throw || it.return); + fn?.call(it, e); + cb && cb(null); + } + private _pull(size: number, it: SourceIterator<T>) { + const bm = this._bytesMode; + let r: IteratorResult<T> | null = null; + while (this.readable && !(r = it.next(bm ? size : null)).done) { + if (size != null) { + size -= (bm && ArrayBuffer.isView(r.value) ? r.value.byteLength : 1); + } + if (!this.push(r.value) || size <= 0) { break; } + } + if ((r?.done || !this.readable) && (this.push(null) || true)) { + it.return && it.return(); + } + return !this.readable; + } +} + +/** @ignore */ +class AsyncIterableReadable<T extends Uint8Array | any> extends Readable { + private _pulling: boolean; + private _bytesMode: boolean; + private _iterator: AsyncSourceIterator<T>; + constructor(it: AsyncSourceIterator<T>, options?: ReadableOptions) { + super(options); + this._iterator = it; + this._pulling = false; + this._bytesMode = !options || !options.objectMode; + } + _read(size: number) { + const it = this._iterator; + if (it && !this._pulling && (this._pulling = true)) { + (async () => this._pulling = await this._pull(size, it))(); + } + } + _destroy(e: Error | null, cb: (e: Error | null) => void) { + const it = this._iterator; + let fn: any; + it && (fn = e != null && it.throw || it.return); + fn?.call(it, e).then(() => cb && cb(null)) || (cb && cb(null)); + } + private async _pull(size: number, it: AsyncSourceIterator<T>) { + const bm = this._bytesMode; + let r: IteratorResult<T> | null = null; + while (this.readable && !(r = await it.next(bm ? size : null)).done) { + if (size != null) { + size -= (bm && ArrayBuffer.isView(r.value) ? r.value.byteLength : 1); + } + if (!this.push(r.value) || size <= 0) { break; } + } + if ((r?.done || !this.readable) && (this.push(null) || true)) { + it.return && it.return(); + } + return !this.readable; + } +} diff --git a/src/arrow/js/src/io/node/reader.ts b/src/arrow/js/src/io/node/reader.ts new file mode 100644 index 000000000..a51fb0b40 --- /dev/null +++ b/src/arrow/js/src/io/node/reader.ts @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { DataType } from '../../type'; +import { Duplex, DuplexOptions } from 'stream'; +import { RecordBatch } from '../../recordbatch'; +import { AsyncByteQueue } from '../../io/stream'; +import { RecordBatchReader } from '../../ipc/reader'; + +/** @ignore */ +export function recordBatchReaderThroughNodeStream<T extends { [key: string]: DataType } = any>(options?: DuplexOptions & { autoDestroy: boolean }) { + return new RecordBatchReaderDuplex<T>(options); +} + +/** @ignore */ +type CB = (error?: Error | null | undefined) => void; + +/** @ignore */ +class RecordBatchReaderDuplex<T extends { [key: string]: DataType } = any> extends Duplex { + private _pulling = false; + private _autoDestroy = true; + private _reader: RecordBatchReader | null; + private _asyncQueue: AsyncByteQueue | null; + constructor(options?: DuplexOptions & { autoDestroy: boolean }) { + super({ allowHalfOpen: false, ...options, readableObjectMode: true, writableObjectMode: false }); + this._reader = null; + this._pulling = false; + this._asyncQueue = new AsyncByteQueue(); + this._autoDestroy = options && (typeof options.autoDestroy === 'boolean') ? options.autoDestroy : true; + } + _final(cb?: CB) { + const aq = this._asyncQueue; + aq?.close(); + cb && cb(); + } + _write(x: any, _: string, cb: CB) { + const aq = this._asyncQueue; + aq?.write(x); + cb && cb(); + return true; + } + _read(size: number) { + const aq = this._asyncQueue; + if (aq && !this._pulling && (this._pulling = true)) { + (async () => { + if (!this._reader) { + this._reader = await this._open(aq); + } + this._pulling = await this._pull(size, this._reader); + })(); + } + } + _destroy(err: Error | null, cb: (error: Error | null) => void) { + const aq = this._asyncQueue; + if (aq) { err ? aq.abort(err) : aq.close(); } + cb(this._asyncQueue = this._reader = null); + } + async _open(source: AsyncByteQueue) { + return await (await RecordBatchReader.from<T>(source)).open({ autoDestroy: this._autoDestroy }); + } + async _pull(size: number, reader: RecordBatchReader<T>) { + let r: IteratorResult<RecordBatch<T>> | null = null; + while (this.readable && !(r = await reader.next()).done) { + if (!this.push(r.value) || (size != null && --size <= 0)) { break; } + } + if (!this.readable || (r?.done && (reader.autoDestroy || (await reader.reset().open()).closed))) { + this.push(null); + await reader.cancel(); + } + return !this.readable; + } +} diff --git a/src/arrow/js/src/io/node/writer.ts b/src/arrow/js/src/io/node/writer.ts new file mode 100644 index 000000000..79d61b9a3 --- /dev/null +++ b/src/arrow/js/src/io/node/writer.ts @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { DataType } from '../../type'; +import { Duplex, DuplexOptions } from 'stream'; +import { AsyncByteStream } from '../../io/stream'; +import { RecordBatchWriter } from '../../ipc/writer'; + +/** @ignore */ +export function recordBatchWriterThroughNodeStream<T extends { [key: string]: DataType } = any>(this: typeof RecordBatchWriter, options?: DuplexOptions & { autoDestroy: boolean }) { + return new RecordBatchWriterDuplex(new this<T>(options)); +} + +/** @ignore */ +type CB = (error?: Error | null | undefined) => void; + +/** @ignore */ +class RecordBatchWriterDuplex<T extends { [key: string]: DataType } = any> extends Duplex { + private _pulling = false; + private _reader: AsyncByteStream | null; + private _writer: RecordBatchWriter | null; + constructor(writer: RecordBatchWriter<T>, options?: DuplexOptions) { + super({ allowHalfOpen: false, ...options, writableObjectMode: true, readableObjectMode: false }); + this._writer = writer; + this._reader = new AsyncByteStream(writer); + } + _final(cb?: CB) { + const writer = this._writer; + writer?.close(); + cb && cb(); + } + _write(x: any, _: string, cb: CB) { + const writer = this._writer; + writer?.write(x); + cb && cb(); + return true; + } + _read(size: number) { + const it = this._reader; + if (it && !this._pulling && (this._pulling = true)) { + (async () => this._pulling = await this._pull(size, it))(); + } + } + _destroy(err: Error | null, cb: (error: Error | null) => void) { + const writer = this._writer; + if (writer) { err ? writer.abort(err) : writer.close(); } + cb(this._reader = this._writer = null); + } + async _pull(size: number, reader: AsyncByteStream) { + let r: IteratorResult<Uint8Array> | null = null; + while (this.readable && !(r = await reader.next(size || null)).done) { + if (size != null && r.value) { + size -= r.value.byteLength; + } + if (!this.push(r.value) || size <= 0) { break; } + } + if ((r?.done || !this.readable)) { + this.push(null); + await reader.cancel(); + } + return !this.readable; + } +} diff --git a/src/arrow/js/src/io/stream.ts b/src/arrow/js/src/io/stream.ts new file mode 100644 index 000000000..2384ab0b9 --- /dev/null +++ b/src/arrow/js/src/io/stream.ts @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import streamAdapters from './adapters'; +import { decodeUtf8 } from '../util/utf8'; +import { ITERATOR_DONE, Readable, Writable, AsyncQueue } from './interfaces'; +import { toUint8Array, joinUint8Arrays, ArrayBufferViewInput } from '../util/buffer'; + +import { + isPromise, isFetchResponse, + isIterable, isAsyncIterable, + isReadableDOMStream, isReadableNodeStream +} from '../util/compat'; + +/** @ignore */ +export type WritableSink<T> = Writable<T> | WritableStream<T> | NodeJS.WritableStream | null; +/** @ignore */ +export type ReadableSource<T> = Readable<T> | PromiseLike<T> | AsyncIterable<T> | ReadableStream<T> | NodeJS.ReadableStream | null; + +/** @ignore */ +export class AsyncByteQueue<T extends ArrayBufferViewInput = Uint8Array> extends AsyncQueue<Uint8Array, T> { + public write(value: ArrayBufferViewInput | Uint8Array) { + if ((value = toUint8Array(value)).byteLength > 0) { + return super.write(value as T); + } + } + public toString(sync: true): string; + public toString(sync?: false): Promise<string>; + public toString(sync = false) { + return sync + ? decodeUtf8(this.toUint8Array(true)) + : this.toUint8Array(false).then(decodeUtf8); + } + public toUint8Array(sync: true): Uint8Array; + public toUint8Array(sync?: false): Promise<Uint8Array>; + public toUint8Array(sync = false) { + return sync ? joinUint8Arrays(this._values as any[])[0] : (async () => { + const buffers = []; + let byteLength = 0; + for await (const chunk of this) { + buffers.push(chunk); + byteLength += chunk.byteLength; + } + return joinUint8Arrays(buffers, byteLength)[0]; + })(); + } +} + +/** @ignore */ +export class ByteStream implements IterableIterator<Uint8Array> { + private source!: ByteStreamSource<Uint8Array>; + constructor(source?: Iterable<ArrayBufferViewInput> | ArrayBufferViewInput) { + if (source) { + this.source = new ByteStreamSource(streamAdapters.fromIterable(source)); + } + } + [Symbol.iterator]() { return this; } + public next(value?: any) { return this.source.next(value); } + public throw(value?: any) { return this.source.throw(value); } + public return(value?: any) { return this.source.return(value); } + public peek(size?: number | null) { return this.source.peek(size); } + public read(size?: number | null) { return this.source.read(size); } +} + +/** @ignore */ +export class AsyncByteStream implements Readable<Uint8Array>, AsyncIterableIterator<Uint8Array> { + private source!: AsyncByteStreamSource<Uint8Array>; + constructor(source?: PromiseLike<ArrayBufferViewInput> | Response | ReadableStream<ArrayBufferViewInput> | NodeJS.ReadableStream | AsyncIterable<ArrayBufferViewInput> | Iterable<ArrayBufferViewInput>) { + if (source instanceof AsyncByteStream) { + this.source = (source as AsyncByteStream).source; + } else if (source instanceof AsyncByteQueue) { + this.source = new AsyncByteStreamSource(streamAdapters.fromAsyncIterable(source)); + } else if (isReadableNodeStream(source)) { + this.source = new AsyncByteStreamSource(streamAdapters.fromNodeStream(source)); + } else if (isReadableDOMStream<ArrayBufferViewInput>(source)) { + this.source = new AsyncByteStreamSource(streamAdapters.fromDOMStream(source)); + } else if (isFetchResponse(source)) { + this.source = new AsyncByteStreamSource(streamAdapters.fromDOMStream(source.body!)); + } else if (isIterable<ArrayBufferViewInput>(source)) { + this.source = new AsyncByteStreamSource(streamAdapters.fromIterable(source)); + } else if (isPromise<ArrayBufferViewInput>(source)) { + this.source = new AsyncByteStreamSource(streamAdapters.fromAsyncIterable(source)); + } else if (isAsyncIterable<ArrayBufferViewInput>(source)) { + this.source = new AsyncByteStreamSource(streamAdapters.fromAsyncIterable(source)); + } + } + [Symbol.asyncIterator]() { return this; } + public next(value?: any) { return this.source.next(value); } + public throw(value?: any) { return this.source.throw(value); } + public return(value?: any) { return this.source.return(value); } + public get closed(): Promise<void> { return this.source.closed; } + public cancel(reason?: any) { return this.source.cancel(reason); } + public peek(size?: number | null) { return this.source.peek(size); } + public read(size?: number | null) { return this.source.read(size); } +} + +/** @ignore */ +type ByteStreamSourceIterator<T> = Generator<T, null, { cmd: 'peek' | 'read'; size?: number | null }>; +/** @ignore */ +type AsyncByteStreamSourceIterator<T> = AsyncGenerator<T, null, { cmd: 'peek' | 'read'; size?: number | null }>; + +/** @ignore */ +class ByteStreamSource<T> { + constructor(protected source: ByteStreamSourceIterator<T>) {} + public cancel(reason?: any) { this.return(reason); } + public peek(size?: number | null): T | null { return this.next(size, 'peek').value; } + public read(size?: number | null): T | null { return this.next(size, 'read').value; } + public next(size?: number | null, cmd: 'peek' | 'read' = 'read') { return this.source.next({ cmd, size }); } + public throw(value?: any) { return Object.create((this.source.throw && this.source.throw(value)) || ITERATOR_DONE); } + public return(value?: any) { return Object.create((this.source.return && this.source.return(value)) || ITERATOR_DONE); } +} + +/** @ignore */ +class AsyncByteStreamSource<T> implements Readable<T> { + + private _closedPromise: Promise<void>; + private _closedPromiseResolve?: (value?: any) => void; + constructor (protected source: ByteStreamSourceIterator<T> | AsyncByteStreamSourceIterator<T>) { + this._closedPromise = new Promise((r) => this._closedPromiseResolve = r); + } + public async cancel(reason?: any) { await this.return(reason); } + public get closed(): Promise<void> { return this._closedPromise; } + public async read(size?: number | null): Promise<T | null> { return (await this.next(size, 'read')).value; } + public async peek(size?: number | null): Promise<T | null> { return (await this.next(size, 'peek')).value; } + public async next(size?: number | null, cmd: 'peek' | 'read' = 'read') { return (await this.source.next({ cmd, size })); } + public async throw(value?: any) { + const result = (this.source.throw && await this.source.throw(value)) || ITERATOR_DONE; + this._closedPromiseResolve && this._closedPromiseResolve(); + this._closedPromiseResolve = undefined; + return Object.create(result); + } + public async return(value?: any) { + const result = (this.source.return && await this.source.return(value)) || ITERATOR_DONE; + this._closedPromiseResolve && this._closedPromiseResolve(); + this._closedPromiseResolve = undefined; + return Object.create(result); + } +} diff --git a/src/arrow/js/src/io/whatwg/builder.ts b/src/arrow/js/src/io/whatwg/builder.ts new file mode 100644 index 000000000..c65511844 --- /dev/null +++ b/src/arrow/js/src/io/whatwg/builder.ts @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { DataType } from '../../type'; +import { Vector } from '../../vector'; +import { VectorType as V } from '../../interfaces'; +import { Builder, BuilderOptions } from '../../builder/index'; + +/** @ignore */ +export interface BuilderTransformOptions<T extends DataType = any, TNull = any> extends BuilderOptions<T, TNull> { + queueingStrategy?: 'bytes' | 'count'; + dictionaryHashFunction?: (value: any) => string | number; + readableStrategy?: { highWaterMark?: number; size?: any; type?: 'bytes' }; + writableStrategy?: { highWaterMark?: number; size?: any; type?: 'bytes' }; + valueToChildTypeId?: (builder: Builder<T, TNull>, value: any, offset: number) => number; +} + +/** @ignore */ +export function builderThroughDOMStream<T extends DataType = any, TNull = any>(options: BuilderTransformOptions<T, TNull>) { + return new BuilderTransform(options); +} + +/** @ignore */ +export class BuilderTransform<T extends DataType = any, TNull = any> { + + public readable: ReadableStream<V<T>>; + public writable: WritableStream<T['TValue'] | TNull>; + public _controller: ReadableStreamDefaultController<V<T>> | null; + + private _numChunks = 0; + private _finished = false; + private _bufferedSize = 0; + private _builder: Builder<T, TNull>; + private _getSize: (builder: Builder<T, TNull>) => number; + + constructor(options: BuilderTransformOptions<T, TNull>) { + + // Access properties by string indexers to defeat closure compiler + + const { + ['readableStrategy']: readableStrategy, + ['writableStrategy']: writableStrategy, + ['queueingStrategy']: queueingStrategy = 'count', + ...builderOptions + } = options; + + this._controller = null; + this._builder = Builder.new<T, TNull>(builderOptions); + this._getSize = queueingStrategy !== 'bytes' ? chunkLength : chunkByteLength; + + const { ['highWaterMark']: readableHighWaterMark = queueingStrategy === 'bytes' ? 2 ** 14 : 1000 } = { ...readableStrategy }; + const { ['highWaterMark']: writableHighWaterMark = queueingStrategy === 'bytes' ? 2 ** 14 : 1000 } = { ...writableStrategy }; + + this['readable'] = new ReadableStream<V<T>>({ + ['cancel']: () => { this._builder.clear(); }, + ['pull']: (c) => { this._maybeFlush(this._builder, this._controller = c); }, + ['start']: (c) => { this._maybeFlush(this._builder, this._controller = c); }, + }, { + 'highWaterMark': readableHighWaterMark, + 'size': queueingStrategy !== 'bytes' ? chunkLength : chunkByteLength, + }); + + this['writable'] = new WritableStream({ + ['abort']: () => { this._builder.clear(); }, + ['write']: () => { this._maybeFlush(this._builder, this._controller); }, + ['close']: () => { this._maybeFlush(this._builder.finish(), this._controller); }, + }, { + 'highWaterMark': writableHighWaterMark, + 'size': (value: T['TValue'] | TNull) => this._writeValueAndReturnChunkSize(value), + }); + } + + private _writeValueAndReturnChunkSize(value: T['TValue'] | TNull) { + const bufferedSize = this._bufferedSize; + this._bufferedSize = this._getSize(this._builder.append(value)); + return this._bufferedSize - bufferedSize; + } + + private _maybeFlush(builder: Builder<T, TNull>, controller: ReadableStreamDefaultController<V<T>> | null) { + if (controller === null) { return; } + if (this._bufferedSize >= controller.desiredSize!) { + ++this._numChunks && this._enqueue(controller, builder.toVector()); + } + if (builder.finished) { + if (builder.length > 0 || this._numChunks === 0) { + ++this._numChunks && this._enqueue(controller, builder.toVector()); + } + if (!this._finished && (this._finished = true)) { + this._enqueue(controller, null); + } + } + } + + private _enqueue(controller: ReadableStreamDefaultController<V<T>>, chunk: V<T> | null) { + this._bufferedSize = 0; + this._controller = null; + chunk === null ? controller.close() : controller.enqueue(chunk); + } +} + +/** @ignore */ const chunkLength = <T extends DataType = any>(chunk: Vector<T> | Builder<T>) => chunk.length; +/** @ignore */ const chunkByteLength = <T extends DataType = any>(chunk: Vector<T> | Builder<T>) => chunk.byteLength; diff --git a/src/arrow/js/src/io/whatwg/iterable.ts b/src/arrow/js/src/io/whatwg/iterable.ts new file mode 100644 index 000000000..ce9e97369 --- /dev/null +++ b/src/arrow/js/src/io/whatwg/iterable.ts @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { toUint8Array } from '../../util/buffer'; +import { ReadableDOMStreamOptions } from '../../io/interfaces'; +import { isIterable, isAsyncIterable } from '../../util/compat'; + +/** @ignore */ +type SourceIterator<T> = Generator<T, void, number | null>; +/** @ignore */ +type AsyncSourceIterator<T> = AsyncGenerator<T, void, number | null>; + +/** @ignore */ +export function toDOMStream<T>(source: Iterable<T> | AsyncIterable<T>, options?: ReadableDOMStreamOptions): ReadableStream<T> { + if (isAsyncIterable<T>(source)) { return asyncIterableAsReadableDOMStream(source, options); } + if (isIterable<T>(source)) { return iterableAsReadableDOMStream(source, options); } + /* istanbul ignore next */ + throw new Error(`toDOMStream() must be called with an Iterable or AsyncIterable`); +} + +/** @ignore */ +function iterableAsReadableDOMStream<T>(source: Iterable<T>, options?: ReadableDOMStreamOptions) { + + let it: SourceIterator<T> | null = null; + const bm = (options?.type === 'bytes') || false; + const hwm = options?.highWaterMark || (2 ** 24); + + return new ReadableStream<T>({ + ...options as any, + start(controller) { next(controller, it || (it = source[Symbol.iterator]() as SourceIterator<T>)); }, + pull(controller) { it ? (next(controller, it)) : controller.close(); }, + cancel() { (it?.return && it.return() || true) && (it = null); } + }, { highWaterMark: bm ? hwm : undefined, ...options }); + + function next(controller: ReadableStreamDefaultController<T>, it: SourceIterator<T>) { + let buf: Uint8Array; + let r: IteratorResult<T> | null = null; + let size = controller.desiredSize || null; + while (!(r = it.next(bm ? size : null)).done) { + if (ArrayBuffer.isView(r.value) && (buf = toUint8Array(r.value))) { + size != null && bm && (size = size - buf.byteLength + 1); + r.value = <any> buf; + } + controller.enqueue(r.value); + if (size != null && --size <= 0) { return; } + } + controller.close(); + } +} + +/** @ignore */ +function asyncIterableAsReadableDOMStream<T>(source: AsyncIterable<T>, options?: ReadableDOMStreamOptions) { + + let it: AsyncSourceIterator<T> | null = null; + const bm = (options?.type === 'bytes') || false; + const hwm = options?.highWaterMark || (2 ** 24); + + return new ReadableStream<T>({ + ...options as any, + async start(controller) { await next(controller, it || (it = source[Symbol.asyncIterator]() as AsyncSourceIterator<T>)); }, + async pull(controller) { it ? (await next(controller, it)) : controller.close(); }, + async cancel() { (it?.return && await it.return() || true) && (it = null); }, + }, { highWaterMark: bm ? hwm : undefined, ...options }); + + async function next(controller: ReadableStreamDefaultController<T>, it: AsyncSourceIterator<T>) { + let buf: Uint8Array; + let r: IteratorResult<T> | null = null; + let size = controller.desiredSize || null; + while (!(r = await it.next(bm ? size : null)).done) { + if (ArrayBuffer.isView(r.value) && (buf = toUint8Array(r.value))) { + size != null && bm && (size = size - buf.byteLength + 1); + r.value = <any> buf; + } + controller.enqueue(r.value); + if (size != null && --size <= 0) { return; } + } + controller.close(); + } +} diff --git a/src/arrow/js/src/io/whatwg/reader.ts b/src/arrow/js/src/io/whatwg/reader.ts new file mode 100644 index 000000000..9e19bac53 --- /dev/null +++ b/src/arrow/js/src/io/whatwg/reader.ts @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { DataType } from '../../type'; +import { RecordBatch } from '../../recordbatch'; +import { AsyncByteQueue } from '../../io/stream'; +import { RecordBatchReader } from '../../ipc/reader'; + +/** @ignore */ +export function recordBatchReaderThroughDOMStream<T extends { [key: string]: DataType } = any>(writableStrategy?: ByteLengthQueuingStrategy, readableStrategy?: { autoDestroy: boolean }) { + + const queue = new AsyncByteQueue(); + let reader: RecordBatchReader<T> | null = null; + + const readable = new ReadableStream<RecordBatch<T>>({ + async cancel() { await queue.close(); }, + async start(controller) { await next(controller, reader || (reader = await open())); }, + async pull(controller) { reader ? await next(controller, reader) : controller.close(); } + }); + + return { writable: new WritableStream(queue, { 'highWaterMark': 2 ** 14, ...writableStrategy }), readable }; + + async function open() { + return await (await RecordBatchReader.from<T>(queue)).open(readableStrategy); + } + + async function next(controller: ReadableStreamDefaultController<RecordBatch<T>>, reader: RecordBatchReader<T>) { + let size = controller.desiredSize; + let r: IteratorResult<RecordBatch<T>> | null = null; + while (!(r = await reader.next()).done) { + controller.enqueue(r.value); + if (size != null && --size <= 0) { + return; + } + } + controller.close(); + } +} diff --git a/src/arrow/js/src/io/whatwg/writer.ts b/src/arrow/js/src/io/whatwg/writer.ts new file mode 100644 index 000000000..49789bdd3 --- /dev/null +++ b/src/arrow/js/src/io/whatwg/writer.ts @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { DataType } from '../../type'; +import { RecordBatch } from '../../recordbatch'; +import { AsyncByteStream } from '../../io/stream'; +import { RecordBatchWriter } from '../../ipc/writer'; + +/** @ignore */ +export function recordBatchWriterThroughDOMStream<T extends { [key: string]: DataType } = any>( + this: typeof RecordBatchWriter, + writableStrategy?: QueuingStrategy<RecordBatch<T>> & { autoDestroy: boolean }, + readableStrategy?: { highWaterMark?: number; size?: any } +) { + + const writer = new this<T>(writableStrategy); + const reader = new AsyncByteStream(writer); + const readable = new ReadableStream({ + type: 'bytes', + async cancel() { await reader.cancel(); }, + async pull(controller) { await next(controller); }, + async start(controller) { await next(controller); }, + }, { 'highWaterMark': 2 ** 14, ...readableStrategy }); + + return { writable: new WritableStream(writer, writableStrategy), readable }; + + async function next(controller: ReadableStreamDefaultController<Uint8Array>) { + let buf: Uint8Array | null = null; + let size = controller.desiredSize; + while (buf = await reader.read(size || null)) { + controller.enqueue(buf); + if (size != null && (size -= buf.byteLength) <= 0) { return; } + } + controller.close(); + } +} diff --git a/src/arrow/js/src/ipc/message.ts b/src/arrow/js/src/ipc/message.ts new file mode 100644 index 000000000..34c0aa308 --- /dev/null +++ b/src/arrow/js/src/ipc/message.ts @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { MessageHeader } from '../enum'; +import { flatbuffers } from 'flatbuffers'; +import ByteBuffer = flatbuffers.ByteBuffer; +import { Message } from './metadata/message'; +import { isFileHandle } from '../util/compat'; +import { AsyncRandomAccessFile } from '../io/file'; +import { toUint8Array, ArrayBufferViewInput } from '../util/buffer'; +import { ByteStream, ReadableSource, AsyncByteStream } from '../io/stream'; +import { ArrowJSON, ArrowJSONLike, ITERATOR_DONE, FileHandle } from '../io/interfaces'; + +/** @ignore */ const invalidMessageType = (type: MessageHeader) => `Expected ${MessageHeader[type]} Message in stream, but was null or length 0.`; +/** @ignore */ const nullMessage = (type: MessageHeader) => `Header pointer of flatbuffer-encoded ${MessageHeader[type]} Message is null or length 0.`; +/** @ignore */ const invalidMessageMetadata = (expected: number, actual: number) => `Expected to read ${expected} metadata bytes, but only read ${actual}.`; +/** @ignore */ const invalidMessageBodyLength = (expected: number, actual: number) => `Expected to read ${expected} bytes for message body, but only read ${actual}.`; + +/** @ignore */ +export class MessageReader implements IterableIterator<Message> { + protected source: ByteStream; + constructor(source: ByteStream | ArrayBufferViewInput | Iterable<ArrayBufferViewInput>) { + this.source = source instanceof ByteStream ? source : new ByteStream(source); + } + public [Symbol.iterator](): IterableIterator<Message> { return this as IterableIterator<Message>; } + public next(): IteratorResult<Message> { + let r; + if ((r = this.readMetadataLength()).done) { return ITERATOR_DONE; } + // ARROW-6313: If the first 4 bytes are continuation indicator (-1), read + // the next 4 for the 32-bit metadata length. Otherwise, assume this is a + // pre-v0.15 message, where the first 4 bytes are the metadata length. + if ((r.value === -1) && + (r = this.readMetadataLength()).done) { return ITERATOR_DONE; } + if ((r = this.readMetadata(r.value)).done) { return ITERATOR_DONE; } + return (<any> r) as IteratorResult<Message>; + } + public throw(value?: any) { return this.source.throw(value); } + public return(value?: any) { return this.source.return(value); } + public readMessage<T extends MessageHeader>(type?: T | null) { + let r: IteratorResult<Message<T>>; + if ((r = this.next()).done) { return null; } + if ((type != null) && r.value.headerType !== type) { + throw new Error(invalidMessageType(type)); + } + return r.value; + } + public readMessageBody(bodyLength: number): Uint8Array { + if (bodyLength <= 0) { return new Uint8Array(0); } + const buf = toUint8Array(this.source.read(bodyLength)); + if (buf.byteLength < bodyLength) { + throw new Error(invalidMessageBodyLength(bodyLength, buf.byteLength)); + } + // 1. Work around bugs in fs.ReadStream's internal Buffer pooling, see: https://github.com/nodejs/node/issues/24817 + // 2. Work around https://github.com/whatwg/streams/blob/0ebe4b042e467d9876d80ae045de3843092ad797/reference-implementation/lib/helpers.js#L126 + return /* 1. */ (buf.byteOffset % 8 === 0) && + /* 2. */ (buf.byteOffset + buf.byteLength) <= buf.buffer.byteLength ? buf : buf.slice(); + } + public readSchema(throwIfNull = false) { + const type = MessageHeader.Schema; + const message = this.readMessage(type); + const schema = message?.header(); + if (throwIfNull && !schema) { + throw new Error(nullMessage(type)); + } + return schema; + } + protected readMetadataLength(): IteratorResult<number> { + const buf = this.source.read(PADDING); + const bb = buf && new ByteBuffer(buf); + const len = bb?.readInt32(0) || 0; + return { done: len === 0, value: len }; + } + protected readMetadata(metadataLength: number): IteratorResult<Message> { + const buf = this.source.read(metadataLength); + if (!buf) { return ITERATOR_DONE; } + if (buf.byteLength < metadataLength) { + throw new Error(invalidMessageMetadata(metadataLength, buf.byteLength)); + } + return { done: false, value: Message.decode(buf) }; + } +} + +/** @ignore */ +export class AsyncMessageReader implements AsyncIterableIterator<Message> { + protected source: AsyncByteStream; + constructor(source: ReadableSource<Uint8Array>); + constructor(source: FileHandle, byteLength?: number); + constructor(source: any, byteLength?: number) { + this.source = source instanceof AsyncByteStream ? source + : isFileHandle(source) + ? new AsyncRandomAccessFile(source, byteLength!) + : new AsyncByteStream(source); + } + public [Symbol.asyncIterator](): AsyncIterableIterator<Message> { return this as AsyncIterableIterator<Message>; } + public async next(): Promise<IteratorResult<Message>> { + let r; + if ((r = await this.readMetadataLength()).done) { return ITERATOR_DONE; } + // ARROW-6313: If the first 4 bytes are continuation indicator (-1), read + // the next 4 for the 32-bit metadata length. Otherwise, assume this is a + // pre-v0.15 message, where the first 4 bytes are the metadata length. + if ((r.value === -1) && + (r = await this.readMetadataLength()).done) { return ITERATOR_DONE; } + if ((r = await this.readMetadata(r.value)).done) { return ITERATOR_DONE; } + return (<any> r) as IteratorResult<Message>; + } + public async throw(value?: any) { return await this.source.throw(value); } + public async return(value?: any) { return await this.source.return(value); } + public async readMessage<T extends MessageHeader>(type?: T | null) { + let r: IteratorResult<Message<T>>; + if ((r = await this.next()).done) { return null; } + if ((type != null) && r.value.headerType !== type) { + throw new Error(invalidMessageType(type)); + } + return r.value; + } + public async readMessageBody(bodyLength: number): Promise<Uint8Array> { + if (bodyLength <= 0) { return new Uint8Array(0); } + const buf = toUint8Array(await this.source.read(bodyLength)); + if (buf.byteLength < bodyLength) { + throw new Error(invalidMessageBodyLength(bodyLength, buf.byteLength)); + } + // 1. Work around bugs in fs.ReadStream's internal Buffer pooling, see: https://github.com/nodejs/node/issues/24817 + // 2. Work around https://github.com/whatwg/streams/blob/0ebe4b042e467d9876d80ae045de3843092ad797/reference-implementation/lib/helpers.js#L126 + return /* 1. */ (buf.byteOffset % 8 === 0) && + /* 2. */ (buf.byteOffset + buf.byteLength) <= buf.buffer.byteLength ? buf : buf.slice(); + } + public async readSchema(throwIfNull = false) { + const type = MessageHeader.Schema; + const message = await this.readMessage(type); + const schema = message?.header(); + if (throwIfNull && !schema) { + throw new Error(nullMessage(type)); + } + return schema; + } + protected async readMetadataLength(): Promise<IteratorResult<number>> { + const buf = await this.source.read(PADDING); + const bb = buf && new ByteBuffer(buf); + const len = bb?.readInt32(0) || 0; + return { done: len === 0, value: len }; + } + protected async readMetadata(metadataLength: number): Promise<IteratorResult<Message>> { + const buf = await this.source.read(metadataLength); + if (!buf) { return ITERATOR_DONE; } + if (buf.byteLength < metadataLength) { + throw new Error(invalidMessageMetadata(metadataLength, buf.byteLength)); + } + return { done: false, value: Message.decode(buf) }; + } +} + +/** @ignore */ +export class JSONMessageReader extends MessageReader { + private _schema = false; + private _json: ArrowJSON; + private _body: any[] = []; + private _batchIndex = 0; + private _dictionaryIndex = 0; + constructor(source: ArrowJSON | ArrowJSONLike) { + super(new Uint8Array(0)); + this._json = source instanceof ArrowJSON ? source : new ArrowJSON(source); + } + public next() { + const { _json } = this; + if (!this._schema) { + this._schema = true; + const message = Message.fromJSON(_json.schema, MessageHeader.Schema); + return { done: false, value: message }; + } + if (this._dictionaryIndex < _json.dictionaries.length) { + const batch = _json.dictionaries[this._dictionaryIndex++]; + this._body = batch['data']['columns']; + const message = Message.fromJSON(batch, MessageHeader.DictionaryBatch); + return { done: false, value: message }; + } + if (this._batchIndex < _json.batches.length) { + const batch = _json.batches[this._batchIndex++]; + this._body = batch['columns']; + const message = Message.fromJSON(batch, MessageHeader.RecordBatch); + return { done: false, value: message }; + } + this._body = []; + return ITERATOR_DONE; + } + public readMessageBody(_bodyLength?: number) { + return flattenDataSources(this._body) as any; + function flattenDataSources(xs: any[]): any[][] { + return (xs || []).reduce<any[][]>((buffers, column: any) => [ + ...buffers, + ...(column['VALIDITY'] && [column['VALIDITY']] || []), + ...(column['TYPE'] && [column['TYPE']] || []), + ...(column['OFFSET'] && [column['OFFSET']] || []), + ...(column['DATA'] && [column['DATA']] || []), + ...flattenDataSources(column['children']) + ], [] as any[][]); + } + } + public readMessage<T extends MessageHeader>(type?: T | null) { + let r: IteratorResult<Message<T>>; + if ((r = this.next()).done) { return null; } + if ((type != null) && r.value.headerType !== type) { + throw new Error(invalidMessageType(type)); + } + return r.value; + } + public readSchema() { + const type = MessageHeader.Schema; + const message = this.readMessage(type); + const schema = message?.header(); + if (!message || !schema) { + throw new Error(nullMessage(type)); + } + return schema; + } +} + +/** @ignore */ +export const PADDING = 4; +/** @ignore */ +export const MAGIC_STR = 'ARROW1'; +/** @ignore */ +export const MAGIC = new Uint8Array(MAGIC_STR.length); + +for (let i = 0; i < MAGIC_STR.length; i += 1 | 0) { + MAGIC[i] = MAGIC_STR.charCodeAt(i); +} + +/** @ignore */ +export function checkForMagicArrowString(buffer: Uint8Array, index = 0) { + for (let i = -1, n = MAGIC.length; ++i < n;) { + if (MAGIC[i] !== buffer[index + i]) { + return false; + } + } + return true; +} + +/** @ignore */ +export const magicLength = MAGIC.length; +/** @ignore */ +export const magicAndPadding = magicLength + PADDING; +/** @ignore */ +export const magicX2AndPadding = magicLength * 2 + PADDING; diff --git a/src/arrow/js/src/ipc/metadata/file.ts b/src/arrow/js/src/ipc/metadata/file.ts new file mode 100644 index 000000000..5a1be844e --- /dev/null +++ b/src/arrow/js/src/ipc/metadata/file.ts @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* eslint-disable @typescript-eslint/naming-convention */ + +import { + Block as _Block, + Footer as _Footer +} from '../../fb/File'; + +import { flatbuffers } from 'flatbuffers'; + +import Long = flatbuffers.Long; +import Builder = flatbuffers.Builder; +import ByteBuffer = flatbuffers.ByteBuffer; + +import { Schema } from '../../schema'; +import { MetadataVersion } from '../../enum'; +import { toUint8Array } from '../../util/buffer'; +import { ArrayBufferViewInput } from '../../util/buffer'; + +/** @ignore */ +class Footer_ { + + /** @nocollapse */ + public static decode(buf: ArrayBufferViewInput) { + buf = new ByteBuffer(toUint8Array(buf)); + const footer = _Footer.getRootAsFooter(buf); + const schema = Schema.decode(footer.schema()!); + return new OffHeapFooter(schema, footer) as Footer_; + } + + /** @nocollapse */ + public static encode(footer: Footer_) { + + const b: Builder = new Builder(); + const schemaOffset = Schema.encode(b, footer.schema); + + _Footer.startRecordBatchesVector(b, footer.numRecordBatches); + [...footer.recordBatches()].slice().reverse().forEach((rb) => FileBlock.encode(b, rb)); + const recordBatchesOffset = b.endVector(); + + _Footer.startDictionariesVector(b, footer.numDictionaries); + [...footer.dictionaryBatches()].slice().reverse().forEach((db) => FileBlock.encode(b, db)); + + const dictionaryBatchesOffset = b.endVector(); + + _Footer.startFooter(b); + _Footer.addSchema(b, schemaOffset); + _Footer.addVersion(b, MetadataVersion.V4); + _Footer.addRecordBatches(b, recordBatchesOffset); + _Footer.addDictionaries(b, dictionaryBatchesOffset); + _Footer.finishFooterBuffer(b, _Footer.endFooter(b)); + + return b.asUint8Array(); + } + + protected _recordBatches!: FileBlock[]; + protected _dictionaryBatches!: FileBlock[]; + public get numRecordBatches() { return this._recordBatches.length; } + public get numDictionaries() { return this._dictionaryBatches.length; } + + constructor(public schema: Schema, + public version: MetadataVersion = MetadataVersion.V4, + recordBatches?: FileBlock[], dictionaryBatches?: FileBlock[]) { + recordBatches && (this._recordBatches = recordBatches); + dictionaryBatches && (this._dictionaryBatches = dictionaryBatches); + } + + public *recordBatches(): Iterable<FileBlock> { + for (let block, i = -1, n = this.numRecordBatches; ++i < n;) { + if (block = this.getRecordBatch(i)) { yield block; } + } + } + + public *dictionaryBatches(): Iterable<FileBlock> { + for (let block, i = -1, n = this.numDictionaries; ++i < n;) { + if (block = this.getDictionaryBatch(i)) { yield block; } + } + } + + public getRecordBatch(index: number) { + return index >= 0 + && index < this.numRecordBatches + && this._recordBatches[index] || null; + } + + public getDictionaryBatch(index: number) { + return index >= 0 + && index < this.numDictionaries + && this._dictionaryBatches[index] || null; + } +} + +export { Footer_ as Footer }; + +/** @ignore */ +class OffHeapFooter extends Footer_ { + + public get numRecordBatches() { return this._footer.recordBatchesLength(); } + public get numDictionaries() { return this._footer.dictionariesLength(); } + + constructor(schema: Schema, protected _footer: _Footer) { + super(schema, _footer.version()); + } + + public getRecordBatch(index: number) { + if (index >= 0 && index < this.numRecordBatches) { + const fileBlock = this._footer.recordBatches(index); + if (fileBlock) { return FileBlock.decode(fileBlock); } + } + return null; + } + + public getDictionaryBatch(index: number) { + if (index >= 0 && index < this.numDictionaries) { + const fileBlock = this._footer.dictionaries(index); + if (fileBlock) { return FileBlock.decode(fileBlock); } + } + return null; + } +} + +/** @ignore */ +export class FileBlock { + + /** @nocollapse */ + public static decode(block: _Block) { + return new FileBlock(block.metaDataLength(), block.bodyLength(), block.offset()); + } + + /** @nocollapse */ + public static encode(b: Builder, fileBlock: FileBlock) { + const { metaDataLength } = fileBlock; + const offset = new Long(fileBlock.offset, 0); + const bodyLength = new Long(fileBlock.bodyLength, 0); + return _Block.createBlock(b, offset, metaDataLength, bodyLength); + } + + public offset: number; + public bodyLength: number; + public metaDataLength: number; + + constructor(metaDataLength: number, bodyLength: Long | number, offset: Long | number) { + this.metaDataLength = metaDataLength; + this.offset = typeof offset === 'number' ? offset : offset.low; + this.bodyLength = typeof bodyLength === 'number' ? bodyLength : bodyLength.low; + } +} diff --git a/src/arrow/js/src/ipc/metadata/json.ts b/src/arrow/js/src/ipc/metadata/json.ts new file mode 100644 index 000000000..399615c31 --- /dev/null +++ b/src/arrow/js/src/ipc/metadata/json.ts @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* eslint-disable brace-style */ + +import { Schema, Field } from '../../schema'; +import { + DataType, Dictionary, TimeBitWidth, + Utf8, Binary, Decimal, FixedSizeBinary, + List, FixedSizeList, Map_, Struct, Union, + Bool, Null, Int, Float, Date_, Time, Interval, Timestamp, IntBitWidth, Int32, TKeys, +} from '../../type'; + +import { DictionaryBatch, RecordBatch, FieldNode, BufferRegion } from './message'; +import { TimeUnit, Precision, IntervalUnit, UnionMode, DateUnit } from '../../enum'; + +/** @ignore */ +export function schemaFromJSON(_schema: any, dictionaries: Map<number, DataType> = new Map()) { + return new Schema( + schemaFieldsFromJSON(_schema, dictionaries), + customMetadataFromJSON(_schema['customMetadata']), + dictionaries + ); +} + +/** @ignore */ +export function recordBatchFromJSON(b: any) { + return new RecordBatch( + b['count'], + fieldNodesFromJSON(b['columns']), + buffersFromJSON(b['columns']) + ); +} + +/** @ignore */ +export function dictionaryBatchFromJSON(b: any) { + return new DictionaryBatch( + recordBatchFromJSON(b['data']), + b['id'], b['isDelta'] + ); +} + +/** @ignore */ +function schemaFieldsFromJSON(_schema: any, dictionaries?: Map<number, DataType>) { + return (_schema['fields'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries)); +} + +/** @ignore */ +function fieldChildrenFromJSON(_field: any, dictionaries?: Map<number, DataType>): Field[] { + return (_field['children'] || []).filter(Boolean).map((f: any) => Field.fromJSON(f, dictionaries)); +} + +/** @ignore */ +function fieldNodesFromJSON(xs: any[]): FieldNode[] { + return (xs || []).reduce<FieldNode[]>((fieldNodes, column: any) => [ + ...fieldNodes, + new FieldNode( + column['count'], + nullCountFromJSON(column['VALIDITY']) + ), + ...fieldNodesFromJSON(column['children']) + ], [] as FieldNode[]); +} + +/** @ignore */ +function buffersFromJSON(xs: any[], buffers: BufferRegion[] = []): BufferRegion[] { + for (let i = -1, n = (xs || []).length; ++i < n;) { + const column = xs[i]; + column['VALIDITY'] && buffers.push(new BufferRegion(buffers.length, column['VALIDITY'].length)); + column['TYPE'] && buffers.push(new BufferRegion(buffers.length, column['TYPE'].length)); + column['OFFSET'] && buffers.push(new BufferRegion(buffers.length, column['OFFSET'].length)); + column['DATA'] && buffers.push(new BufferRegion(buffers.length, column['DATA'].length)); + buffers = buffersFromJSON(column['children'], buffers); + } + return buffers; +} + +/** @ignore */ +function nullCountFromJSON(validity: number[]) { + return (validity || []).reduce((sum, val) => sum + +(val === 0), 0); +} + +/** @ignore */ +export function fieldFromJSON(_field: any, dictionaries?: Map<number, DataType>) { + + let id: number; + let keys: TKeys | null; + let field: Field | void; + let dictMeta: any; + let type: DataType<any>; + let dictType: Dictionary; + + // If no dictionary encoding + if (!dictionaries || !(dictMeta = _field['dictionary'])) { + type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries)); + field = new Field(_field['name'], type, _field['nullable'], customMetadataFromJSON(_field['customMetadata'])); + } + // If dictionary encoded and the first time we've seen this dictionary id, decode + // the data type and child fields, then wrap in a Dictionary type and insert the + // data type into the dictionary types map. + else if (!dictionaries.has(id = dictMeta['id'])) { + // a dictionary index defaults to signed 32 bit int if unspecified + keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32(); + dictionaries.set(id, type = typeFromJSON(_field, fieldChildrenFromJSON(_field, dictionaries))); + dictType = new Dictionary(type, keys, id, dictMeta['isOrdered']); + field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata'])); + } + // If dictionary encoded, and have already seen this dictionary Id in the schema, then reuse the + // data type and wrap in a new Dictionary type and field. + else { + // a dictionary index defaults to signed 32 bit int if unspecified + keys = (keys = dictMeta['indexType']) ? indexTypeFromJSON(keys) as TKeys : new Int32(); + dictType = new Dictionary(dictionaries.get(id)!, keys, id, dictMeta['isOrdered']); + field = new Field(_field['name'], dictType, _field['nullable'], customMetadataFromJSON(_field['customMetadata'])); + } + return field || null; +} + +/** @ignore */ +function customMetadataFromJSON(_metadata?: Record<string, string>) { + return new Map<string, string>(Object.entries(_metadata || {})); +} + +/** @ignore */ +function indexTypeFromJSON(_type: any) { + return new Int(_type['isSigned'], _type['bitWidth']); +} + +/** @ignore */ +function typeFromJSON(f: any, children?: Field[]): DataType<any> { + + const typeId = f['type']['name']; + + switch (typeId) { + case 'NONE': return new Null(); + case 'null': return new Null(); + case 'binary': return new Binary(); + case 'utf8': return new Utf8(); + case 'bool': return new Bool(); + case 'list': return new List((children || [])[0]); + case 'struct': return new Struct(children || []); + case 'struct_': return new Struct(children || []); + } + + switch (typeId) { + case 'int': { + const t = f['type']; + return new Int(t['isSigned'], t['bitWidth'] as IntBitWidth); + } + case 'floatingpoint': { + const t = f['type']; + return new Float(Precision[t['precision']] as any); + } + case 'decimal': { + const t = f['type']; + return new Decimal(t['scale'], t['precision']); + } + case 'date': { + const t = f['type']; + return new Date_(DateUnit[t['unit']] as any); + } + case 'time': { + const t = f['type']; + return new Time(TimeUnit[t['unit']] as any, t['bitWidth'] as TimeBitWidth); + } + case 'timestamp': { + const t = f['type']; + return new Timestamp(TimeUnit[t['unit']] as any, t['timezone']); + } + case 'interval': { + const t = f['type']; + return new Interval(IntervalUnit[t['unit']] as any); + } + case 'union': { + const t = f['type']; + return new Union(UnionMode[t['mode']] as any, (t['typeIds'] || []), children || []); + } + case 'fixedsizebinary': { + const t = f['type']; + return new FixedSizeBinary(t['byteWidth']); + } + case 'fixedsizelist': { + const t = f['type']; + return new FixedSizeList(t['listSize'], (children || [])[0]); + } + case 'map': { + const t = f['type']; + return new Map_((children || [])[0], t['keysSorted']); + } + } + throw new Error(`Unrecognized type: "${typeId}"`); +} diff --git a/src/arrow/js/src/ipc/metadata/message.ts b/src/arrow/js/src/ipc/metadata/message.ts new file mode 100644 index 000000000..2ebb73e4c --- /dev/null +++ b/src/arrow/js/src/ipc/metadata/message.ts @@ -0,0 +1,621 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* eslint-disable brace-style */ + +import { flatbuffers } from 'flatbuffers'; + +import { + Type, + Int as _Int, + Field as _Field, + Schema as _Schema, + Buffer as _Buffer, + KeyValue as _KeyValue, + Endianness as _Endianness, + DictionaryEncoding as _DictionaryEncoding, + FloatingPoint as _FloatingPoint, + Decimal as _Decimal, + Date as _Date, + Time as _Time, + Timestamp as _Timestamp, + Interval as _Interval, + Union as _Union, + FixedSizeBinary as _FixedSizeBinary, + FixedSizeList as _FixedSizeList, + Map as _Map, +} from '../../fb/Schema'; + +import { + Message as _Message, + FieldNode as _FieldNode, + RecordBatch as _RecordBatch, + DictionaryBatch as _DictionaryBatch, +} from '../../fb/Message'; + +import { Schema, Field } from '../../schema'; +import { toUint8Array } from '../../util/buffer'; +import { ArrayBufferViewInput } from '../../util/buffer'; +import { MessageHeader, MetadataVersion } from '../../enum'; +import { instance as typeAssembler } from '../../visitor/typeassembler'; +import { fieldFromJSON, schemaFromJSON, recordBatchFromJSON, dictionaryBatchFromJSON } from './json'; + +import Long = flatbuffers.Long; +import Builder = flatbuffers.Builder; +import ByteBuffer = flatbuffers.ByteBuffer; + +import { + DataType, Dictionary, TimeBitWidth, + Utf8, Binary, Decimal, FixedSizeBinary, + List, FixedSizeList, Map_, Struct, Union, + Bool, Null, Int, Float, Date_, Time, Interval, Timestamp, IntBitWidth, Int32, TKeys, +} from '../../type'; + +/** + * @ignore + * @private + **/ +export class Message<T extends MessageHeader = any> { + + /** @nocollapse */ + public static fromJSON<T extends MessageHeader>(msg: any, headerType: T): Message<T> { + const message = new Message(0, MetadataVersion.V4, headerType); + message._createHeader = messageHeaderFromJSON(msg, headerType); + return message; + } + + /** @nocollapse */ + public static decode(buf: ArrayBufferViewInput) { + buf = new ByteBuffer(toUint8Array(buf)); + const _message = _Message.getRootAsMessage(buf); + const bodyLength: Long = _message.bodyLength()!; + const version: MetadataVersion = _message.version(); + const headerType: MessageHeader = _message.headerType(); + const message = new Message(bodyLength, version, headerType); + message._createHeader = decodeMessageHeader(_message, headerType); + return message; + } + + /** @nocollapse */ + public static encode<T extends MessageHeader>(message: Message<T>) { + const b = new Builder(); + let headerOffset = -1; + if (message.isSchema()) { + headerOffset = Schema.encode(b, message.header() as Schema); + } else if (message.isRecordBatch()) { + headerOffset = RecordBatch.encode(b, message.header() as RecordBatch); + } else if (message.isDictionaryBatch()) { + headerOffset = DictionaryBatch.encode(b, message.header() as DictionaryBatch); + } + _Message.startMessage(b); + _Message.addVersion(b, MetadataVersion.V4); + _Message.addHeader(b, headerOffset); + _Message.addHeaderType(b, message.headerType); + _Message.addBodyLength(b, new Long(message.bodyLength, 0)); + _Message.finishMessageBuffer(b, _Message.endMessage(b)); + return b.asUint8Array(); + } + + /** @nocollapse */ + public static from(header: Schema | RecordBatch | DictionaryBatch, bodyLength = 0) { + if (header instanceof Schema) { + return new Message(0, MetadataVersion.V4, MessageHeader.Schema, header); + } + if (header instanceof RecordBatch) { + return new Message(bodyLength, MetadataVersion.V4, MessageHeader.RecordBatch, header); + } + if (header instanceof DictionaryBatch) { + return new Message(bodyLength, MetadataVersion.V4, MessageHeader.DictionaryBatch, header); + } + throw new Error(`Unrecognized Message header: ${header}`); + } + + public body: Uint8Array; + protected _headerType: T; + protected _bodyLength: number; + protected _version: MetadataVersion; + public get type() { return this.headerType; } + public get version() { return this._version; } + public get headerType() { return this._headerType; } + public get bodyLength() { return this._bodyLength; } + protected _createHeader!: MessageHeaderDecoder; + public header() { return this._createHeader<T>(); } + public isSchema(): this is Message<MessageHeader.Schema> { return this.headerType === MessageHeader.Schema; } + public isRecordBatch(): this is Message<MessageHeader.RecordBatch> { return this.headerType === MessageHeader.RecordBatch; } + public isDictionaryBatch(): this is Message<MessageHeader.DictionaryBatch> { return this.headerType === MessageHeader.DictionaryBatch; } + + constructor(bodyLength: Long | number, version: MetadataVersion, headerType: T, header?: any) { + this._version = version; + this._headerType = headerType; + this.body = new Uint8Array(0); + header && (this._createHeader = () => header); + this._bodyLength = typeof bodyLength === 'number' ? bodyLength : bodyLength.low; + } +} + +/** + * @ignore + * @private + **/ +export class RecordBatch { + protected _length: number; + protected _nodes: FieldNode[]; + protected _buffers: BufferRegion[]; + public get nodes() { return this._nodes; } + public get length() { return this._length; } + public get buffers() { return this._buffers; } + constructor(length: Long | number, nodes: FieldNode[], buffers: BufferRegion[]) { + this._nodes = nodes; + this._buffers = buffers; + this._length = typeof length === 'number' ? length : length.low; + } +} + +/** + * @ignore + * @private + **/ +export class DictionaryBatch { + + protected _id: number; + protected _isDelta: boolean; + protected _data: RecordBatch; + public get id() { return this._id; } + public get data() { return this._data; } + public get isDelta() { return this._isDelta; } + public get length(): number { return this.data.length; } + public get nodes(): FieldNode[] { return this.data.nodes; } + public get buffers(): BufferRegion[] { return this.data.buffers; } + + constructor(data: RecordBatch, id: Long | number, isDelta = false) { + this._data = data; + this._isDelta = isDelta; + this._id = typeof id === 'number' ? id : id.low; + } +} + +/** + * @ignore + * @private + **/ +export class BufferRegion { + public offset: number; + public length: number; + constructor(offset: Long | number, length: Long | number) { + this.offset = typeof offset === 'number' ? offset : offset.low; + this.length = typeof length === 'number' ? length : length.low; + } +} + +/** + * @ignore + * @private + **/ +export class FieldNode { + public length: number; + public nullCount: number; + constructor(length: Long | number, nullCount: Long | number) { + this.length = typeof length === 'number' ? length : length.low; + this.nullCount = typeof nullCount === 'number' ? nullCount : nullCount.low; + } +} + +/** @ignore */ +function messageHeaderFromJSON(message: any, type: MessageHeader) { + return (() => { + switch (type) { + case MessageHeader.Schema: return Schema.fromJSON(message); + case MessageHeader.RecordBatch: return RecordBatch.fromJSON(message); + case MessageHeader.DictionaryBatch: return DictionaryBatch.fromJSON(message); + } + throw new Error(`Unrecognized Message type: { name: ${MessageHeader[type]}, type: ${type} }`); + }) as MessageHeaderDecoder; +} + +/** @ignore */ +function decodeMessageHeader(message: _Message, type: MessageHeader) { + return (() => { + switch (type) { + case MessageHeader.Schema: return Schema.decode(message.header(new _Schema())!); + case MessageHeader.RecordBatch: return RecordBatch.decode(message.header(new _RecordBatch())!, message.version()); + case MessageHeader.DictionaryBatch: return DictionaryBatch.decode(message.header(new _DictionaryBatch())!, message.version()); + } + throw new Error(`Unrecognized Message type: { name: ${MessageHeader[type]}, type: ${type} }`); + }) as MessageHeaderDecoder; +} + +Field['encode'] = encodeField; +Field['decode'] = decodeField; +Field['fromJSON'] = fieldFromJSON; + +Schema['encode'] = encodeSchema; +Schema['decode'] = decodeSchema; +Schema['fromJSON'] = schemaFromJSON; + +RecordBatch['encode'] = encodeRecordBatch; +RecordBatch['decode'] = decodeRecordBatch; +RecordBatch['fromJSON'] = recordBatchFromJSON; + +DictionaryBatch['encode'] = encodeDictionaryBatch; +DictionaryBatch['decode'] = decodeDictionaryBatch; +DictionaryBatch['fromJSON'] = dictionaryBatchFromJSON; + +FieldNode['encode'] = encodeFieldNode; +FieldNode['decode'] = decodeFieldNode; + +BufferRegion['encode'] = encodeBufferRegion; +BufferRegion['decode'] = decodeBufferRegion; + +declare module '../../schema' { + namespace Field { + export { encodeField as encode }; + export { decodeField as decode }; + export { fieldFromJSON as fromJSON }; + } + namespace Schema { + export { encodeSchema as encode }; + export { decodeSchema as decode }; + export { schemaFromJSON as fromJSON }; + } +} + +declare module './message' { + namespace RecordBatch { + export { encodeRecordBatch as encode }; + export { decodeRecordBatch as decode }; + export { recordBatchFromJSON as fromJSON }; + } + namespace DictionaryBatch { + export { encodeDictionaryBatch as encode }; + export { decodeDictionaryBatch as decode }; + export { dictionaryBatchFromJSON as fromJSON }; + } + namespace FieldNode { + export { encodeFieldNode as encode }; + export { decodeFieldNode as decode }; + } + namespace BufferRegion { + export { encodeBufferRegion as encode }; + export { decodeBufferRegion as decode }; + } +} + +/** @ignore */ +function decodeSchema(_schema: _Schema, dictionaries: Map<number, DataType> = new Map()) { + const fields = decodeSchemaFields(_schema, dictionaries); + return new Schema(fields, decodeCustomMetadata(_schema), dictionaries); +} + +/** @ignore */ +function decodeRecordBatch(batch: _RecordBatch, version = MetadataVersion.V4) { + return new RecordBatch(batch.length(), decodeFieldNodes(batch), decodeBuffers(batch, version)); +} + +/** @ignore */ +function decodeDictionaryBatch(batch: _DictionaryBatch, version = MetadataVersion.V4) { + return new DictionaryBatch(RecordBatch.decode(batch.data()!, version), batch.id(), batch.isDelta()); +} + +/** @ignore */ +function decodeBufferRegion(b: _Buffer) { + return new BufferRegion(b.offset(), b.length()); +} + +/** @ignore */ +function decodeFieldNode(f: _FieldNode) { + return new FieldNode(f.length(), f.nullCount()); +} + +/** @ignore */ +function decodeFieldNodes(batch: _RecordBatch) { + const nodes = [] as FieldNode[]; + for (let f, i = -1, j = -1, n = batch.nodesLength(); ++i < n;) { + if (f = batch.nodes(i)) { + nodes[++j] = FieldNode.decode(f); + } + } + return nodes; +} + +/** @ignore */ +function decodeBuffers(batch: _RecordBatch, version: MetadataVersion) { + const bufferRegions = [] as BufferRegion[]; + for (let b, i = -1, j = -1, n = batch.buffersLength(); ++i < n;) { + if (b = batch.buffers(i)) { + // If this Arrow buffer was written before version 4, + // advance the buffer's bb_pos 8 bytes to skip past + // the now-removed page_id field + if (version < MetadataVersion.V4) { + b.bb_pos += (8 * (i + 1)); + } + bufferRegions[++j] = BufferRegion.decode(b); + } + } + return bufferRegions; +} + +/** @ignore */ +function decodeSchemaFields(schema: _Schema, dictionaries?: Map<number, DataType>) { + const fields = [] as Field[]; + for (let f, i = -1, j = -1, n = schema.fieldsLength(); ++i < n;) { + if (f = schema.fields(i)) { + fields[++j] = Field.decode(f, dictionaries); + } + } + return fields; +} + +/** @ignore */ +function decodeFieldChildren(field: _Field, dictionaries?: Map<number, DataType>): Field[] { + const children = [] as Field[]; + for (let f, i = -1, j = -1, n = field.childrenLength(); ++i < n;) { + if (f = field.children(i)) { + children[++j] = Field.decode(f, dictionaries); + } + } + return children; +} + +/** @ignore */ +function decodeField(f: _Field, dictionaries?: Map<number, DataType>) { + + let id: number; + let field: Field | void; + let type: DataType<any>; + let keys: _Int | TKeys | null; + let dictType: Dictionary; + let dictMeta: _DictionaryEncoding | null; + + // If no dictionary encoding + if (!dictionaries || !(dictMeta = f.dictionary())) { + type = decodeFieldType(f, decodeFieldChildren(f, dictionaries)); + field = new Field(f.name()!, type, f.nullable(), decodeCustomMetadata(f)); + } + // If dictionary encoded and the first time we've seen this dictionary id, decode + // the data type and child fields, then wrap in a Dictionary type and insert the + // data type into the dictionary types map. + else if (!dictionaries.has(id = dictMeta.id().low)) { + // a dictionary index defaults to signed 32 bit int if unspecified + keys = (keys = dictMeta.indexType()) ? decodeIndexType(keys) as TKeys : new Int32(); + dictionaries.set(id, type = decodeFieldType(f, decodeFieldChildren(f, dictionaries))); + dictType = new Dictionary(type, keys, id, dictMeta.isOrdered()); + field = new Field(f.name()!, dictType, f.nullable(), decodeCustomMetadata(f)); + } + // If dictionary encoded, and have already seen this dictionary Id in the schema, then reuse the + // data type and wrap in a new Dictionary type and field. + else { + // a dictionary index defaults to signed 32 bit int if unspecified + keys = (keys = dictMeta.indexType()) ? decodeIndexType(keys) as TKeys : new Int32(); + dictType = new Dictionary(dictionaries.get(id)!, keys, id, dictMeta.isOrdered()); + field = new Field(f.name()!, dictType, f.nullable(), decodeCustomMetadata(f)); + } + return field || null; +} + +/** @ignore */ +function decodeCustomMetadata(parent?: _Schema | _Field | null) { + const data = new Map<string, string>(); + if (parent) { + for (let entry, key, i = -1, n = parent.customMetadataLength() | 0; ++i < n;) { + if ((entry = parent.customMetadata(i)) && (key = entry.key()) != null) { + data.set(key, entry.value()!); + } + } + } + return data; +} + +/** @ignore */ +function decodeIndexType(_type: _Int) { + return new Int(_type.isSigned(), _type.bitWidth() as IntBitWidth); +} + +/** @ignore */ +function decodeFieldType(f: _Field, children?: Field[]): DataType<any> { + + const typeId = f.typeType(); + + switch (typeId) { + case Type['NONE']: return new Null(); + case Type['Null']: return new Null(); + case Type['Binary']: return new Binary(); + case Type['Utf8']: return new Utf8(); + case Type['Bool']: return new Bool(); + case Type['List']: return new List((children || [])[0]); + case Type['Struct_']: return new Struct(children || []); + } + + switch (typeId) { + case Type['Int']: { + const t = f.type(new _Int())!; + return new Int(t.isSigned(), t.bitWidth()); + } + case Type['FloatingPoint']: { + const t = f.type(new _FloatingPoint())!; + return new Float(t.precision()); + } + case Type['Decimal']: { + const t = f.type(new _Decimal())!; + return new Decimal(t.scale(), t.precision()); + } + case Type['Date']: { + const t = f.type(new _Date())!; + return new Date_(t.unit()); + } + case Type['Time']: { + const t = f.type(new _Time())!; + return new Time(t.unit(), t.bitWidth() as TimeBitWidth); + } + case Type['Timestamp']: { + const t = f.type(new _Timestamp())!; + return new Timestamp(t.unit(), t.timezone()); + } + case Type['Interval']: { + const t = f.type(new _Interval())!; + return new Interval(t.unit()); + } + case Type['Union']: { + const t = f.type(new _Union())!; + return new Union(t.mode(), t.typeIdsArray() || [], children || []); + } + case Type['FixedSizeBinary']: { + const t = f.type(new _FixedSizeBinary())!; + return new FixedSizeBinary(t.byteWidth()); + } + case Type['FixedSizeList']: { + const t = f.type(new _FixedSizeList())!; + return new FixedSizeList(t.listSize(), (children || [])[0]); + } + case Type['Map']: { + const t = f.type(new _Map())!; + return new Map_((children || [])[0], t.keysSorted()); + } + } + throw new Error(`Unrecognized type: "${Type[typeId]}" (${typeId})`); +} + +/** @ignore */ +function encodeSchema(b: Builder, schema: Schema) { + + const fieldOffsets = schema.fields.map((f) => Field.encode(b, f)); + + _Schema.startFieldsVector(b, fieldOffsets.length); + + const fieldsVectorOffset = _Schema.createFieldsVector(b, fieldOffsets); + + const metadataOffset = !(schema.metadata && schema.metadata.size > 0) ? -1 : + _Schema.createCustomMetadataVector(b, [...schema.metadata].map(([k, v]) => { + const key = b.createString(`${k}`); + const val = b.createString(`${v}`); + _KeyValue.startKeyValue(b); + _KeyValue.addKey(b, key); + _KeyValue.addValue(b, val); + return _KeyValue.endKeyValue(b); + })); + + _Schema.startSchema(b); + _Schema.addFields(b, fieldsVectorOffset); + _Schema.addEndianness(b, platformIsLittleEndian ? _Endianness.Little : _Endianness.Big); + + if (metadataOffset !== -1) { _Schema.addCustomMetadata(b, metadataOffset); } + + return _Schema.endSchema(b); +} + +/** @ignore */ +function encodeField(b: Builder, field: Field) { + + let nameOffset = -1; + let typeOffset = -1; + let dictionaryOffset = -1; + + const type = field.type; + let typeId: Type = <any> field.typeId; + + if (!DataType.isDictionary(type)) { + typeOffset = typeAssembler.visit(type, b)!; + } else { + typeId = type.dictionary.typeId; + dictionaryOffset = typeAssembler.visit(type, b)!; + typeOffset = typeAssembler.visit(type.dictionary, b)!; + } + + const childOffsets = (type.children || []).map((f: Field) => Field.encode(b, f)); + const childrenVectorOffset = _Field.createChildrenVector(b, childOffsets); + + const metadataOffset = !(field.metadata && field.metadata.size > 0) ? -1 : + _Field.createCustomMetadataVector(b, [...field.metadata].map(([k, v]) => { + const key = b.createString(`${k}`); + const val = b.createString(`${v}`); + _KeyValue.startKeyValue(b); + _KeyValue.addKey(b, key); + _KeyValue.addValue(b, val); + return _KeyValue.endKeyValue(b); + })); + + if (field.name) { + nameOffset = b.createString(field.name); + } + + _Field.startField(b); + _Field.addType(b, typeOffset); + _Field.addTypeType(b, typeId); + _Field.addChildren(b, childrenVectorOffset); + _Field.addNullable(b, !!field.nullable); + + if (nameOffset !== -1) { _Field.addName(b, nameOffset); } + if (dictionaryOffset !== -1) { _Field.addDictionary(b, dictionaryOffset); } + if (metadataOffset !== -1) { _Field.addCustomMetadata(b, metadataOffset); } + + return _Field.endField(b); +} + +/** @ignore */ +function encodeRecordBatch(b: Builder, recordBatch: RecordBatch) { + + const nodes = recordBatch.nodes || []; + const buffers = recordBatch.buffers || []; + + _RecordBatch.startNodesVector(b, nodes.length); + nodes.slice().reverse().forEach((n) => FieldNode.encode(b, n)); + + const nodesVectorOffset = b.endVector(); + + _RecordBatch.startBuffersVector(b, buffers.length); + buffers.slice().reverse().forEach((b_) => BufferRegion.encode(b, b_)); + + const buffersVectorOffset = b.endVector(); + + _RecordBatch.startRecordBatch(b); + _RecordBatch.addLength(b, new Long(recordBatch.length, 0)); + _RecordBatch.addNodes(b, nodesVectorOffset); + _RecordBatch.addBuffers(b, buffersVectorOffset); + return _RecordBatch.endRecordBatch(b); +} + +/** @ignore */ +function encodeDictionaryBatch(b: Builder, dictionaryBatch: DictionaryBatch) { + const dataOffset = RecordBatch.encode(b, dictionaryBatch.data); + _DictionaryBatch.startDictionaryBatch(b); + _DictionaryBatch.addId(b, new Long(dictionaryBatch.id, 0)); + _DictionaryBatch.addIsDelta(b, dictionaryBatch.isDelta); + _DictionaryBatch.addData(b, dataOffset); + return _DictionaryBatch.endDictionaryBatch(b); +} + +/** @ignore */ +function encodeFieldNode(b: Builder, node: FieldNode) { + return _FieldNode.createFieldNode(b, new Long(node.length, 0), new Long(node.nullCount, 0)); +} + +/** @ignore */ +function encodeBufferRegion(b: Builder, node: BufferRegion) { + return _Buffer.createBuffer(b, new Long(node.offset, 0), new Long(node.length, 0)); +} + +/** @ignore */ +const platformIsLittleEndian = (function() { + const buffer = new ArrayBuffer(2); + new DataView(buffer).setInt16(0, 256, true /* littleEndian */); + // Int16Array uses the platform's endianness. + return new Int16Array(buffer)[0] === 256; +})(); + +/** @ignore */ +type MessageHeaderDecoder = <T extends MessageHeader>() => T extends MessageHeader.Schema ? Schema + : T extends MessageHeader.RecordBatch ? RecordBatch + : T extends MessageHeader.DictionaryBatch ? DictionaryBatch : never; diff --git a/src/arrow/js/src/ipc/reader.ts b/src/arrow/js/src/ipc/reader.ts new file mode 100644 index 000000000..a150ac1bb --- /dev/null +++ b/src/arrow/js/src/ipc/reader.ts @@ -0,0 +1,739 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from '../vector'; +import { DataType } from '../type'; +import { MessageHeader } from '../enum'; +import { Footer } from './metadata/file'; +import { Schema, Field } from '../schema'; +import streamAdapters from '../io/adapters'; +import { Message } from './metadata/message'; +import * as metadata from './metadata/message'; +import { ArrayBufferViewInput } from '../util/buffer'; +import { ByteStream, AsyncByteStream } from '../io/stream'; +import { RandomAccessFile, AsyncRandomAccessFile } from '../io/file'; +import { VectorLoader, JSONVectorLoader } from '../visitor/vectorloader'; +import { RecordBatch, _InternalEmptyPlaceholderRecordBatch } from '../recordbatch'; +import { + FileHandle, + ArrowJSONLike, + ITERATOR_DONE, + ReadableInterop, +} from '../io/interfaces'; +import { + MessageReader, AsyncMessageReader, JSONMessageReader, + checkForMagicArrowString, magicLength, magicAndPadding, magicX2AndPadding +} from './message'; +import { + isPromise, + isIterable, isAsyncIterable, + isIteratorResult, isArrowJSON, + isFileHandle, isFetchResponse, + isReadableDOMStream, isReadableNodeStream +} from '../util/compat'; + +/** @ignore */ export type FromArg0 = ArrowJSONLike; +/** @ignore */ export type FromArg1 = PromiseLike<ArrowJSONLike>; +/** @ignore */ export type FromArg2 = Iterable<ArrayBufferViewInput> | ArrayBufferViewInput; +/** @ignore */ export type FromArg3 = PromiseLike<Iterable<ArrayBufferViewInput> | ArrayBufferViewInput>; +/** @ignore */ export type FromArg4 = Response | NodeJS.ReadableStream | ReadableStream<ArrayBufferViewInput> | AsyncIterable<ArrayBufferViewInput>; +/** @ignore */ export type FromArg5 = FileHandle | PromiseLike<FileHandle> | PromiseLike<FromArg4>; +/** @ignore */ export type FromArgs = FromArg0 | FromArg1 | FromArg2 | FromArg3 | FromArg4 | FromArg5; + +/** @ignore */ type OpenOptions = { autoDestroy?: boolean }; +/** @ignore */ type RecordBatchReaders<T extends { [key: string]: DataType } = any> = RecordBatchFileReader<T> | RecordBatchStreamReader<T>; +/** @ignore */ type AsyncRecordBatchReaders<T extends { [key: string]: DataType } = any> = AsyncRecordBatchFileReader<T> | AsyncRecordBatchStreamReader<T>; +/** @ignore */ type RecordBatchFileReaders<T extends { [key: string]: DataType } = any> = RecordBatchFileReader<T> | AsyncRecordBatchFileReader<T>; +/** @ignore */ type RecordBatchStreamReaders<T extends { [key: string]: DataType } = any> = RecordBatchStreamReader<T> | AsyncRecordBatchStreamReader<T>; + +export class RecordBatchReader<T extends { [key: string]: DataType } = any> extends ReadableInterop<RecordBatch<T>> { + + protected _impl: RecordBatchReaderImpls<T>; + protected constructor(impl: RecordBatchReaderImpls<T>) { + super(); + this._impl = impl; + } + + public get closed() { return this._impl.closed; } + public get schema() { return this._impl.schema; } + public get autoDestroy() { return this._impl.autoDestroy; } + public get dictionaries() { return this._impl.dictionaries; } + public get numDictionaries() { return this._impl.numDictionaries; } + public get numRecordBatches() { return this._impl.numRecordBatches; } + public get footer(): Footer | null { return this._impl.isFile() ? this._impl.footer : null; } + + public isSync(): this is RecordBatchReaders<T> { return this._impl.isSync(); } + public isAsync(): this is AsyncRecordBatchReaders<T> { return this._impl.isAsync(); } + public isFile(): this is RecordBatchFileReaders<T> { return this._impl.isFile(); } + public isStream(): this is RecordBatchStreamReaders<T> { return this._impl.isStream(); } + + public next() { + return this._impl.next(); + } + public throw(value?: any) { + return this._impl.throw(value); + } + public return(value?: any) { + return this._impl.return(value); + } + public cancel() { + return this._impl.cancel(); + } + public reset(schema?: Schema<T> | null): this { + this._impl.reset(schema); + this._DOMStream = undefined; + this._nodeStream = undefined; + return this; + } + public open(options?: OpenOptions) { + const opening = this._impl.open(options); + return isPromise(opening) ? opening.then(() => this) : this; + } + public readRecordBatch(index: number): RecordBatch<T> | null | Promise<RecordBatch<T> | null> { + return this._impl.isFile() ? this._impl.readRecordBatch(index) : null; + } + public [Symbol.iterator](): IterableIterator<RecordBatch<T>> { + return (<IterableIterator<RecordBatch<T>>> this._impl)[Symbol.iterator](); + } + public [Symbol.asyncIterator](): AsyncIterableIterator<RecordBatch<T>> { + return (<AsyncIterableIterator<RecordBatch<T>>> this._impl)[Symbol.asyncIterator](); + } + public toDOMStream() { + return streamAdapters.toDOMStream<RecordBatch<T>>( + (this.isSync() + ? { [Symbol.iterator]: () => this } as Iterable<RecordBatch<T>> + : { [Symbol.asyncIterator]: () => this } as AsyncIterable<RecordBatch<T>>)); + } + public toNodeStream() { + return streamAdapters.toNodeStream<RecordBatch<T>>( + (this.isSync() + ? { [Symbol.iterator]: () => this } as Iterable<RecordBatch<T>> + : { [Symbol.asyncIterator]: () => this } as AsyncIterable<RecordBatch<T>>), + { objectMode: true }); + } + + /** @nocollapse */ + // @ts-ignore + public static throughNode(options?: import('stream').DuplexOptions & { autoDestroy: boolean }): import('stream').Duplex { + throw new Error(`"throughNode" not available in this environment`); + } + /** @nocollapse */ + public static throughDOM<T extends { [key: string]: DataType }>( + // @ts-ignore + writableStrategy?: ByteLengthQueuingStrategy, + // @ts-ignore + readableStrategy?: { autoDestroy: boolean } + ): { writable: WritableStream<Uint8Array>; readable: ReadableStream<RecordBatch<T>> } { + throw new Error(`"throughDOM" not available in this environment`); + } + + public static from<T extends RecordBatchReader>(source: T): T; + public static from<T extends { [key: string]: DataType } = any>(source: FromArg0): RecordBatchStreamReader<T>; + public static from<T extends { [key: string]: DataType } = any>(source: FromArg1): Promise<RecordBatchStreamReader<T>>; + public static from<T extends { [key: string]: DataType } = any>(source: FromArg2): RecordBatchFileReader<T> | RecordBatchStreamReader<T>; + public static from<T extends { [key: string]: DataType } = any>(source: FromArg3): Promise<RecordBatchFileReader<T> | RecordBatchStreamReader<T>>; + public static from<T extends { [key: string]: DataType } = any>(source: FromArg4): Promise<RecordBatchFileReader<T> | AsyncRecordBatchReaders<T>>; + public static from<T extends { [key: string]: DataType } = any>(source: FromArg5): Promise<AsyncRecordBatchFileReader<T> | AsyncRecordBatchStreamReader<T>>; + /** @nocollapse */ + public static from<T extends { [key: string]: DataType } = any>(source: any) { + if (source instanceof RecordBatchReader) { + return source; + } else if (isArrowJSON(source)) { + return fromArrowJSON<T>(source); + } else if (isFileHandle(source)) { + return fromFileHandle<T>(source); + } else if (isPromise<any>(source)) { + return (async () => await RecordBatchReader.from<any>(await source))(); + } else if (isFetchResponse(source) || isReadableDOMStream(source) || isReadableNodeStream(source) || isAsyncIterable(source)) { + return fromAsyncByteStream<T>(new AsyncByteStream(source)); + } + return fromByteStream<T>(new ByteStream(source)); + } + + public static readAll<T extends RecordBatchReader>(source: T): T extends RecordBatchReaders ? IterableIterator<T> : AsyncIterableIterator<T>; + public static readAll<T extends { [key: string]: DataType } = any>(source: FromArg0): IterableIterator<RecordBatchStreamReader<T>>; + public static readAll<T extends { [key: string]: DataType } = any>(source: FromArg1): AsyncIterableIterator<RecordBatchStreamReader<T>>; + public static readAll<T extends { [key: string]: DataType } = any>(source: FromArg2): IterableIterator<RecordBatchFileReader<T> | RecordBatchStreamReader<T>>; + public static readAll<T extends { [key: string]: DataType } = any>(source: FromArg3): AsyncIterableIterator<RecordBatchFileReader<T> | RecordBatchStreamReader<T>>; + public static readAll<T extends { [key: string]: DataType } = any>(source: FromArg4): AsyncIterableIterator<RecordBatchFileReader<T> | AsyncRecordBatchReaders<T>>; + public static readAll<T extends { [key: string]: DataType } = any>(source: FromArg5): AsyncIterableIterator<AsyncRecordBatchFileReader<T> | AsyncRecordBatchStreamReader<T>>; + /** @nocollapse */ + public static readAll<T extends { [key: string]: DataType } = any>(source: any) { + if (source instanceof RecordBatchReader) { + return source.isSync() ? readAllSync(source) : readAllAsync(source as AsyncRecordBatchReaders<T>); + } else if (isArrowJSON(source) || ArrayBuffer.isView(source) || isIterable<ArrayBufferViewInput>(source) || isIteratorResult(source)) { + return readAllSync<T>(source) as IterableIterator<RecordBatchReaders<T>>; + } + return readAllAsync<T>(source) as AsyncIterableIterator<RecordBatchReaders<T> | AsyncRecordBatchReaders<T>>; + } +} + +// +// Since TS is a structural type system, we define the following subclass stubs +// so that concrete types exist to associate with with the interfaces below. +// +// The implementation for each RecordBatchReader is hidden away in the set of +// `RecordBatchReaderImpl` classes in the second half of this file. This allows +// us to export a single RecordBatchReader class, and swap out the impl based +// on the io primitives or underlying arrow (JSON, file, or stream) at runtime. +// +// Async/await makes our job a bit harder, since it forces everything to be +// either fully sync or fully async. This is why the logic for the reader impls +// has been duplicated into both sync and async variants. Since the RBR +// delegates to its impl, an RBR with an AsyncRecordBatchFileReaderImpl for +// example will return async/await-friendly Promises, but one with a (sync) +// RecordBatchStreamReaderImpl will always return values. Nothing should be +// different about their logic, aside from the async handling. This is also why +// this code looks highly structured, as it should be nearly identical and easy +// to follow. +// + +/** @ignore */ +export class RecordBatchStreamReader<T extends { [key: string]: DataType } = any> extends RecordBatchReader<T> { + constructor(protected _impl: RecordBatchStreamReaderImpl<T>) { super (_impl); } + public [Symbol.iterator]() { return (this._impl as IterableIterator<RecordBatch<T>>)[Symbol.iterator](); } + public async *[Symbol.asyncIterator](): AsyncIterableIterator<RecordBatch<T>> { yield* this[Symbol.iterator](); } +} +/** @ignore */ +export class AsyncRecordBatchStreamReader<T extends { [key: string]: DataType } = any> extends RecordBatchReader<T> { + constructor(protected _impl: AsyncRecordBatchStreamReaderImpl<T>) { super (_impl); } + public [Symbol.iterator](): IterableIterator<RecordBatch<T>> { throw new Error(`AsyncRecordBatchStreamReader is not Iterable`); } + public [Symbol.asyncIterator]() { return (this._impl as AsyncIterableIterator<RecordBatch<T>>)[Symbol.asyncIterator](); } +} +/** @ignore */ +export class RecordBatchFileReader<T extends { [key: string]: DataType } = any> extends RecordBatchStreamReader<T> { + constructor(protected _impl: RecordBatchFileReaderImpl<T>) { super (_impl); } +} +/** @ignore */ +export class AsyncRecordBatchFileReader<T extends { [key: string]: DataType } = any> extends AsyncRecordBatchStreamReader<T> { + constructor(protected _impl: AsyncRecordBatchFileReaderImpl<T>) { super (_impl); } +} + +// +// Now override the return types for each sync/async RecordBatchReader variant +// + +/** @ignore */ +export interface RecordBatchStreamReader<T extends { [key: string]: DataType } = any> extends RecordBatchReader<T> { + open(options?: OpenOptions | undefined): this; + cancel(): void; + throw(value?: any): IteratorResult<any>; + return(value?: any): IteratorResult<any>; + next(value?: any): IteratorResult<RecordBatch<T>>; +} + +/** @ignore */ +export interface AsyncRecordBatchStreamReader<T extends { [key: string]: DataType } = any> extends RecordBatchReader<T> { + open(options?: OpenOptions | undefined): Promise<this>; + cancel(): Promise<void>; + throw(value?: any): Promise<IteratorResult<any>>; + return(value?: any): Promise<IteratorResult<any>>; + next(value?: any): Promise<IteratorResult<RecordBatch<T>>>; +} + +/** @ignore */ +export interface RecordBatchFileReader<T extends { [key: string]: DataType } = any> extends RecordBatchStreamReader<T> { + readRecordBatch(index: number): RecordBatch<T> | null; +} + +/** @ignore */ +export interface AsyncRecordBatchFileReader<T extends { [key: string]: DataType } = any> extends AsyncRecordBatchStreamReader<T> { + readRecordBatch(index: number): Promise<RecordBatch<T> | null>; +} + +/** @ignore */ +type RecordBatchReaderImpls<T extends { [key: string]: DataType } = any> = + RecordBatchJSONReaderImpl<T> | + RecordBatchFileReaderImpl<T> | + RecordBatchStreamReaderImpl<T> | + AsyncRecordBatchFileReaderImpl<T> | + AsyncRecordBatchStreamReaderImpl<T>; + +/** @ignore */ +interface RecordBatchReaderImpl<T extends { [key: string]: DataType } = any> { + + closed: boolean; + schema: Schema<T>; + autoDestroy: boolean; + dictionaries: Map<number, Vector>; + + isFile(): this is RecordBatchFileReaders<T>; + isStream(): this is RecordBatchStreamReaders<T>; + isSync(): this is RecordBatchReaders<T>; + isAsync(): this is AsyncRecordBatchReaders<T>; + + reset(schema?: Schema<T> | null): this; +} + +/** @ignore */ +interface RecordBatchStreamReaderImpl<T extends { [key: string]: DataType } = any> extends RecordBatchReaderImpl<T> { + + open(options?: OpenOptions): this; + cancel(): void; + + throw(value?: any): IteratorResult<any>; + return(value?: any): IteratorResult<any>; + next(value?: any): IteratorResult<RecordBatch<T>>; + + [Symbol.iterator](): IterableIterator<RecordBatch<T>>; +} + +/** @ignore */ +interface AsyncRecordBatchStreamReaderImpl<T extends { [key: string]: DataType } = any> extends RecordBatchReaderImpl<T> { + + open(options?: OpenOptions): Promise<this>; + cancel(): Promise<void>; + + throw(value?: any): Promise<IteratorResult<any>>; + return(value?: any): Promise<IteratorResult<any>>; + next(value?: any): Promise<IteratorResult<RecordBatch<T>>>; + + [Symbol.asyncIterator](): AsyncIterableIterator<RecordBatch<T>>; +} + +/** @ignore */ +interface RecordBatchFileReaderImpl<T extends { [key: string]: DataType } = any> extends RecordBatchStreamReaderImpl<T> { + readRecordBatch(index: number): RecordBatch<T> | null; +} + +/** @ignore */ +interface AsyncRecordBatchFileReaderImpl<T extends { [key: string]: DataType } = any> extends AsyncRecordBatchStreamReaderImpl<T> { + readRecordBatch(index: number): Promise<RecordBatch<T> | null>; +} + +/** @ignore */ +abstract class RecordBatchReaderImpl<T extends { [key: string]: DataType } = any> implements RecordBatchReaderImpl<T> { + + public schema!: Schema<T>; + public closed = false; + public autoDestroy = true; + public dictionaries: Map<number, Vector>; + + protected _dictionaryIndex = 0; + protected _recordBatchIndex = 0; + public get numDictionaries() { return this._dictionaryIndex; } + public get numRecordBatches() { return this._recordBatchIndex; } + + constructor(dictionaries = new Map<number, Vector>()) { + this.dictionaries = dictionaries; + } + + public isSync(): this is RecordBatchReaders<T> { return false; } + public isAsync(): this is AsyncRecordBatchReaders<T> { return false; } + public isFile(): this is RecordBatchFileReaders<T> { return false; } + public isStream(): this is RecordBatchStreamReaders<T> { return false; } + + public reset(schema?: Schema<T> | null) { + this._dictionaryIndex = 0; + this._recordBatchIndex = 0; + this.schema = <any> schema; + this.dictionaries = new Map(); + return this; + } + + protected _loadRecordBatch(header: metadata.RecordBatch, body: any) { + return new RecordBatch<T>(this.schema, header.length, this._loadVectors(header, body, this.schema.fields)); + } + protected _loadDictionaryBatch(header: metadata.DictionaryBatch, body: any) { + const { id, isDelta, data } = header; + const { dictionaries, schema } = this; + const dictionary = dictionaries.get(id); + if (isDelta || !dictionary) { + const type = schema.dictionaries.get(id)!; + return (dictionary && isDelta ? dictionary.concat( + Vector.new(this._loadVectors(data, body, [type])[0])) : + Vector.new(this._loadVectors(data, body, [type])[0])) as Vector; + } + return dictionary; + } + protected _loadVectors(header: metadata.RecordBatch, body: any, types: (Field | DataType)[]) { + return new VectorLoader(body, header.nodes, header.buffers, this.dictionaries).visitMany(types); + } +} + +/** @ignore */ +class RecordBatchStreamReaderImpl<T extends { [key: string]: DataType } = any> extends RecordBatchReaderImpl<T> implements IterableIterator<RecordBatch<T>> { + + protected _reader: MessageReader; + protected _handle: ByteStream | ArrowJSONLike; + + constructor(source: ByteStream | ArrowJSONLike, dictionaries?: Map<number, Vector>) { + super(dictionaries); + this._reader = !isArrowJSON(source) + ? new MessageReader(this._handle = source) + : new JSONMessageReader(this._handle = source); + } + + public isSync(): this is RecordBatchReaders<T> { return true; } + public isStream(): this is RecordBatchStreamReaders<T> { return true; } + public [Symbol.iterator](): IterableIterator<RecordBatch<T>> { + return this as IterableIterator<RecordBatch<T>>; + } + public cancel() { + if (!this.closed && (this.closed = true)) { + this.reset()._reader.return(); + this._reader = <any> null; + this.dictionaries = <any> null; + } + } + public open(options?: OpenOptions) { + if (!this.closed) { + this.autoDestroy = shouldAutoDestroy(this, options); + if (!(this.schema || (this.schema = this._reader.readSchema()!))) { + this.cancel(); + } + } + return this; + } + public throw(value?: any): IteratorResult<any> { + if (!this.closed && this.autoDestroy && (this.closed = true)) { + return this.reset()._reader.throw(value); + } + return ITERATOR_DONE; + } + public return(value?: any): IteratorResult<any> { + if (!this.closed && this.autoDestroy && (this.closed = true)) { + return this.reset()._reader.return(value); + } + return ITERATOR_DONE; + } + public next(): IteratorResult<RecordBatch<T>> { + if (this.closed) { return ITERATOR_DONE; } + let message: Message | null; + const { _reader: reader } = this; + while (message = this._readNextMessageAndValidate()) { + if (message.isSchema()) { + this.reset(message.header()); + } else if (message.isRecordBatch()) { + this._recordBatchIndex++; + const header = message.header(); + const buffer = reader.readMessageBody(message.bodyLength); + const recordBatch = this._loadRecordBatch(header, buffer); + return { done: false, value: recordBatch }; + } else if (message.isDictionaryBatch()) { + this._dictionaryIndex++; + const header = message.header(); + const buffer = reader.readMessageBody(message.bodyLength); + const vector = this._loadDictionaryBatch(header, buffer); + this.dictionaries.set(header.id, vector); + } + } + if (this.schema && this._recordBatchIndex === 0) { + this._recordBatchIndex++; + return { done: false, value: new _InternalEmptyPlaceholderRecordBatch<T>(this.schema) }; + } + return this.return(); + } + protected _readNextMessageAndValidate<T extends MessageHeader>(type?: T | null) { + return this._reader.readMessage<T>(type); + } +} + +/** @ignore */ +class AsyncRecordBatchStreamReaderImpl<T extends { [key: string]: DataType } = any> extends RecordBatchReaderImpl<T> implements AsyncIterableIterator<RecordBatch<T>> { + + protected _handle: AsyncByteStream; + protected _reader: AsyncMessageReader; + + constructor(source: AsyncByteStream, dictionaries?: Map<number, Vector>) { + super(dictionaries); + this._reader = new AsyncMessageReader(this._handle = source); + } + public isAsync(): this is AsyncRecordBatchReaders<T> { return true; } + public isStream(): this is RecordBatchStreamReaders<T> { return true; } + public [Symbol.asyncIterator](): AsyncIterableIterator<RecordBatch<T>> { + return this as AsyncIterableIterator<RecordBatch<T>>; + } + public async cancel() { + if (!this.closed && (this.closed = true)) { + await this.reset()._reader.return(); + this._reader = <any> null; + this.dictionaries = <any> null; + } + } + public async open(options?: OpenOptions) { + if (!this.closed) { + this.autoDestroy = shouldAutoDestroy(this, options); + if (!(this.schema || (this.schema = (await this._reader.readSchema())!))) { + await this.cancel(); + } + } + return this; + } + public async throw(value?: any): Promise<IteratorResult<any>> { + if (!this.closed && this.autoDestroy && (this.closed = true)) { + return await this.reset()._reader.throw(value); + } + return ITERATOR_DONE; + } + public async return(value?: any): Promise<IteratorResult<any>> { + if (!this.closed && this.autoDestroy && (this.closed = true)) { + return await this.reset()._reader.return(value); + } + return ITERATOR_DONE; + } + public async next() { + if (this.closed) { return ITERATOR_DONE; } + let message: Message | null; + const { _reader: reader } = this; + while (message = await this._readNextMessageAndValidate()) { + if (message.isSchema()) { + await this.reset(message.header()); + } else if (message.isRecordBatch()) { + this._recordBatchIndex++; + const header = message.header(); + const buffer = await reader.readMessageBody(message.bodyLength); + const recordBatch = this._loadRecordBatch(header, buffer); + return { done: false, value: recordBatch }; + } else if (message.isDictionaryBatch()) { + this._dictionaryIndex++; + const header = message.header(); + const buffer = await reader.readMessageBody(message.bodyLength); + const vector = this._loadDictionaryBatch(header, buffer); + this.dictionaries.set(header.id, vector); + } + } + if (this.schema && this._recordBatchIndex === 0) { + this._recordBatchIndex++; + return { done: false, value: new _InternalEmptyPlaceholderRecordBatch<T>(this.schema) }; + } + return await this.return(); + } + protected async _readNextMessageAndValidate<T extends MessageHeader>(type?: T | null) { + return await this._reader.readMessage<T>(type); + } +} + +/** @ignore */ +class RecordBatchFileReaderImpl<T extends { [key: string]: DataType } = any> extends RecordBatchStreamReaderImpl<T> { + + protected _footer?: Footer; + protected _handle!: RandomAccessFile; + public get footer() { return this._footer!; } + public get numDictionaries() { return this._footer ? this._footer.numDictionaries : 0; } + public get numRecordBatches() { return this._footer ? this._footer.numRecordBatches : 0; } + + constructor(source: RandomAccessFile | ArrayBufferViewInput, dictionaries?: Map<number, Vector>) { + super(source instanceof RandomAccessFile ? source : new RandomAccessFile(source), dictionaries); + } + public isSync(): this is RecordBatchReaders<T> { return true; } + public isFile(): this is RecordBatchFileReaders<T> { return true; } + public open(options?: OpenOptions) { + if (!this.closed && !this._footer) { + this.schema = (this._footer = this._readFooter()).schema; + for (const block of this._footer.dictionaryBatches()) { + block && this._readDictionaryBatch(this._dictionaryIndex++); + } + } + return super.open(options); + } + public readRecordBatch(index: number) { + if (this.closed) { return null; } + if (!this._footer) { this.open(); } + const block = this._footer && this._footer.getRecordBatch(index); + if (block && this._handle.seek(block.offset)) { + const message = this._reader.readMessage(MessageHeader.RecordBatch); + if (message?.isRecordBatch()) { + const header = message.header(); + const buffer = this._reader.readMessageBody(message.bodyLength); + const recordBatch = this._loadRecordBatch(header, buffer); + return recordBatch; + } + } + return null; + } + protected _readDictionaryBatch(index: number) { + const block = this._footer && this._footer.getDictionaryBatch(index); + if (block && this._handle.seek(block.offset)) { + const message = this._reader.readMessage(MessageHeader.DictionaryBatch); + if (message?.isDictionaryBatch()) { + const header = message.header(); + const buffer = this._reader.readMessageBody(message.bodyLength); + const vector = this._loadDictionaryBatch(header, buffer); + this.dictionaries.set(header.id, vector); + } + } + } + protected _readFooter() { + const { _handle } = this; + const offset = _handle.size - magicAndPadding; + const length = _handle.readInt32(offset); + const buffer = _handle.readAt(offset - length, length); + return Footer.decode(buffer); + } + protected _readNextMessageAndValidate<T extends MessageHeader>(type?: T | null): Message<T> | null { + if (!this._footer) { this.open(); } + if (this._footer && this._recordBatchIndex < this.numRecordBatches) { + const block = this._footer && this._footer.getRecordBatch(this._recordBatchIndex); + if (block && this._handle.seek(block.offset)) { + return this._reader.readMessage(type); + } + } + return null; + } +} + +/** @ignore */ +class AsyncRecordBatchFileReaderImpl<T extends { [key: string]: DataType } = any> extends AsyncRecordBatchStreamReaderImpl<T> + implements AsyncRecordBatchFileReaderImpl<T> { + + protected _footer?: Footer; + protected _handle!: AsyncRandomAccessFile; + public get footer() { return this._footer!; } + public get numDictionaries() { return this._footer ? this._footer.numDictionaries : 0; } + public get numRecordBatches() { return this._footer ? this._footer.numRecordBatches : 0; } + + constructor(source: FileHandle, byteLength?: number, dictionaries?: Map<number, Vector>); + constructor(source: FileHandle | AsyncRandomAccessFile, dictionaries?: Map<number, Vector>); + constructor(source: FileHandle | AsyncRandomAccessFile, ...rest: any[]) { + const byteLength = typeof rest[0] !== 'number' ? <number> rest.shift() : undefined; + const dictionaries = rest[0] instanceof Map ? <Map<number, Vector>> rest.shift() : undefined; + super(source instanceof AsyncRandomAccessFile ? source : new AsyncRandomAccessFile(source, byteLength), dictionaries); + } + public isFile(): this is RecordBatchFileReaders<T> { return true; } + public isAsync(): this is AsyncRecordBatchReaders<T> { return true; } + public async open(options?: OpenOptions) { + if (!this.closed && !this._footer) { + this.schema = (this._footer = await this._readFooter()).schema; + for (const block of this._footer.dictionaryBatches()) { + block && await this._readDictionaryBatch(this._dictionaryIndex++); + } + } + return await super.open(options); + } + public async readRecordBatch(index: number) { + if (this.closed) { return null; } + if (!this._footer) { await this.open(); } + const block = this._footer && this._footer.getRecordBatch(index); + if (block && (await this._handle.seek(block.offset))) { + const message = await this._reader.readMessage(MessageHeader.RecordBatch); + if (message?.isRecordBatch()) { + const header = message.header(); + const buffer = await this._reader.readMessageBody(message.bodyLength); + const recordBatch = this._loadRecordBatch(header, buffer); + return recordBatch; + } + } + return null; + } + protected async _readDictionaryBatch(index: number) { + const block = this._footer && this._footer.getDictionaryBatch(index); + if (block && (await this._handle.seek(block.offset))) { + const message = await this._reader.readMessage(MessageHeader.DictionaryBatch); + if (message?.isDictionaryBatch()) { + const header = message.header(); + const buffer = await this._reader.readMessageBody(message.bodyLength); + const vector = this._loadDictionaryBatch(header, buffer); + this.dictionaries.set(header.id, vector); + } + } + } + protected async _readFooter() { + const { _handle } = this; + _handle._pending && await _handle._pending; + const offset = _handle.size - magicAndPadding; + const length = await _handle.readInt32(offset); + const buffer = await _handle.readAt(offset - length, length); + return Footer.decode(buffer); + } + protected async _readNextMessageAndValidate<T extends MessageHeader>(type?: T | null): Promise<Message<T> | null> { + if (!this._footer) { await this.open(); } + if (this._footer && this._recordBatchIndex < this.numRecordBatches) { + const block = this._footer.getRecordBatch(this._recordBatchIndex); + if (block && await this._handle.seek(block.offset)) { + return await this._reader.readMessage(type); + } + } + return null; + } +} + +/** @ignore */ +class RecordBatchJSONReaderImpl<T extends { [key: string]: DataType } = any> extends RecordBatchStreamReaderImpl<T> { + constructor(source: ArrowJSONLike, dictionaries?: Map<number, Vector>) { + super(source, dictionaries); + } + protected _loadVectors(header: metadata.RecordBatch, body: any, types: (Field | DataType)[]) { + return new JSONVectorLoader(body, header.nodes, header.buffers, this.dictionaries).visitMany(types); + } +} + +// +// Define some helper functions and static implementations down here. There's +// a bit of branching in the static methods that can lead to the same routines +// being executed, so we've broken those out here for readability. +// + +/** @ignore */ +function shouldAutoDestroy(self: { autoDestroy: boolean }, options?: OpenOptions) { + return options && (typeof options['autoDestroy'] === 'boolean') ? options['autoDestroy'] : self['autoDestroy']; +} + +/** @ignore */ +function* readAllSync<T extends { [key: string]: DataType } = any>(source: RecordBatchReaders<T> | FromArg0 | FromArg2) { + const reader = RecordBatchReader.from<T>(<any> source) as RecordBatchReaders<T>; + try { + if (!reader.open({ autoDestroy: false }).closed) { + do { yield reader; } while (!(reader.reset().open()).closed); + } + } finally { reader.cancel(); } +} + +/** @ignore */ +async function* readAllAsync<T extends { [key: string]: DataType } = any>(source: AsyncRecordBatchReaders<T> | FromArg1 | FromArg3 | FromArg4 | FromArg5) { + const reader = await RecordBatchReader.from<T>(<any> source) as RecordBatchReader<T>; + try { + if (!(await reader.open({ autoDestroy: false })).closed) { + do { yield reader; } while (!(await reader.reset().open()).closed); + } + } finally { await reader.cancel(); } +} + +/** @ignore */ +function fromArrowJSON<T extends { [key: string]: DataType }>(source: ArrowJSONLike) { + return new RecordBatchStreamReader(new RecordBatchJSONReaderImpl<T>(source)); +} + +/** @ignore */ +function fromByteStream<T extends { [key: string]: DataType }>(source: ByteStream) { + const bytes = source.peek((magicLength + 7) & ~7); + return bytes && bytes.byteLength >= 4 ? !checkForMagicArrowString(bytes) + ? new RecordBatchStreamReader(new RecordBatchStreamReaderImpl<T>(source)) + : new RecordBatchFileReader(new RecordBatchFileReaderImpl<T>(source.read())) + : new RecordBatchStreamReader(new RecordBatchStreamReaderImpl<T>(function*(): any {}())); +} + +/** @ignore */ +async function fromAsyncByteStream<T extends { [key: string]: DataType }>(source: AsyncByteStream) { + const bytes = await source.peek((magicLength + 7) & ~7); + return bytes && bytes.byteLength >= 4 ? !checkForMagicArrowString(bytes) + ? new AsyncRecordBatchStreamReader(new AsyncRecordBatchStreamReaderImpl<T>(source)) + : new RecordBatchFileReader(new RecordBatchFileReaderImpl<T>(await source.read())) + : new AsyncRecordBatchStreamReader(new AsyncRecordBatchStreamReaderImpl<T>(async function*(): any {}())); +} + +/** @ignore */ +async function fromFileHandle<T extends { [key: string]: DataType }>(source: FileHandle) { + const { size } = await source.stat(); + const file = new AsyncRandomAccessFile(source, size); + if (size >= magicX2AndPadding) { + if (checkForMagicArrowString(await file.readAt(0, (magicLength + 7) & ~7))) { + return new AsyncRecordBatchFileReader(new AsyncRecordBatchFileReaderImpl<T>(file)); + } + } + return new AsyncRecordBatchStreamReader(new AsyncRecordBatchStreamReaderImpl<T>(file)); +} diff --git a/src/arrow/js/src/ipc/writer.ts b/src/arrow/js/src/ipc/writer.ts new file mode 100644 index 000000000..12aa83355 --- /dev/null +++ b/src/arrow/js/src/ipc/writer.ts @@ -0,0 +1,492 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Table } from '../table'; +import { MAGIC } from './message'; +import { Vector } from '../vector'; +import { Column } from '../column'; +import { DataType } from '../type'; +import { Schema, Field } from '../schema'; +import { Message } from './metadata/message'; +import * as metadata from './metadata/message'; +import { FileBlock, Footer } from './metadata/file'; +import { MessageHeader, MetadataVersion } from '../enum'; +import { compareSchemas } from '../visitor/typecomparator'; +import { WritableSink, AsyncByteQueue } from '../io/stream'; +import { VectorAssembler } from '../visitor/vectorassembler'; +import { JSONTypeAssembler } from '../visitor/jsontypeassembler'; +import { JSONVectorAssembler } from '../visitor/jsonvectorassembler'; +import { ArrayBufferViewInput, toUint8Array } from '../util/buffer'; +import { RecordBatch, _InternalEmptyPlaceholderRecordBatch } from '../recordbatch'; +import { Writable, ReadableInterop, ReadableDOMStreamOptions } from '../io/interfaces'; +import { isPromise, isAsyncIterable, isWritableDOMStream, isWritableNodeStream, isIterable, isObject } from '../util/compat'; + +export interface RecordBatchStreamWriterOptions { + /** + * + */ + autoDestroy?: boolean; + /** + * A flag indicating whether the RecordBatchWriter should construct pre-0.15.0 + * encapsulated IPC Messages, which reserves 4 bytes for the Message metadata + * length instead of 8. + * @see https://issues.apache.org/jira/browse/ARROW-6313 + */ + writeLegacyIpcFormat?: boolean; +} + +export class RecordBatchWriter<T extends { [key: string]: DataType } = any> extends ReadableInterop<Uint8Array> implements Writable<RecordBatch<T>> { + + /** @nocollapse */ + // @ts-ignore + public static throughNode(options?: import('stream').DuplexOptions & { autoDestroy: boolean }): import('stream').Duplex { + throw new Error(`"throughNode" not available in this environment`); + } + /** @nocollapse */ + public static throughDOM<T extends { [key: string]: DataType }>( + // @ts-ignore + writableStrategy?: QueuingStrategy<RecordBatch<T>> & { autoDestroy: boolean }, + // @ts-ignore + readableStrategy?: { highWaterMark?: number; size?: any } + ): { writable: WritableStream<Table<T> | RecordBatch<T>>; readable: ReadableStream<Uint8Array> } { + throw new Error(`"throughDOM" not available in this environment`); + } + + constructor(options?: RecordBatchStreamWriterOptions) { + super(); + isObject(options) || (options = { autoDestroy: true, writeLegacyIpcFormat: false }); + this._autoDestroy = (typeof options.autoDestroy === 'boolean') ? options.autoDestroy : true; + this._writeLegacyIpcFormat = (typeof options.writeLegacyIpcFormat === 'boolean') ? options.writeLegacyIpcFormat : false; + } + + protected _position = 0; + protected _started = false; + protected _autoDestroy: boolean; + protected _writeLegacyIpcFormat: boolean; + // @ts-ignore + protected _sink = new AsyncByteQueue(); + protected _schema: Schema | null = null; + protected _dictionaryBlocks: FileBlock[] = []; + protected _recordBatchBlocks: FileBlock[] = []; + protected _dictionaryDeltaOffsets = new Map<number, number>(); + + public toString(sync: true): string; + public toString(sync?: false): Promise<string>; + public toString(sync: any = false) { + return this._sink.toString(sync) as Promise<string> | string; + } + public toUint8Array(sync: true): Uint8Array; + public toUint8Array(sync?: false): Promise<Uint8Array>; + public toUint8Array(sync: any = false) { + return this._sink.toUint8Array(sync) as Promise<Uint8Array> | Uint8Array; + } + + public writeAll(input: Table<T> | Iterable<RecordBatch<T>>): this; + public writeAll(input: AsyncIterable<RecordBatch<T>>): Promise<this>; + public writeAll(input: PromiseLike<AsyncIterable<RecordBatch<T>>>): Promise<this>; + public writeAll(input: PromiseLike<Table<T> | Iterable<RecordBatch<T>>>): Promise<this>; + public writeAll(input: PromiseLike<any> | Table<T> | Iterable<RecordBatch<T>> | AsyncIterable<RecordBatch<T>>) { + if (isPromise<any>(input)) { + return input.then((x) => this.writeAll(x)); + } else if (isAsyncIterable<RecordBatch<T>>(input)) { + return writeAllAsync(this, input); + } + return writeAll(this, <any> input); + } + + public get closed() { return this._sink.closed; } + public [Symbol.asyncIterator]() { return this._sink[Symbol.asyncIterator](); } + public toDOMStream(options?: ReadableDOMStreamOptions) { return this._sink.toDOMStream(options); } + public toNodeStream(options?: import('stream').ReadableOptions) { return this._sink.toNodeStream(options); } + + public close() { + return this.reset()._sink.close(); + } + public abort(reason?: any) { + return this.reset()._sink.abort(reason); + } + public finish() { + this._autoDestroy ? this.close() : this.reset(this._sink, this._schema); + return this; + } + public reset(sink: WritableSink<ArrayBufferViewInput> = this._sink, schema: Schema<T> | null = null) { + if ((sink === this._sink) || (sink instanceof AsyncByteQueue)) { + this._sink = sink as AsyncByteQueue; + } else { + this._sink = new AsyncByteQueue(); + if (sink && isWritableDOMStream(sink)) { + this.toDOMStream({ type: 'bytes' }).pipeTo(sink); + } else if (sink && isWritableNodeStream(sink)) { + this.toNodeStream({ objectMode: false }).pipe(sink); + } + } + + if (this._started && this._schema) { + this._writeFooter(this._schema); + } + + this._started = false; + this._dictionaryBlocks = []; + this._recordBatchBlocks = []; + this._dictionaryDeltaOffsets = new Map(); + + if (!schema || !(compareSchemas(schema, this._schema))) { + if (schema === null) { + this._position = 0; + this._schema = null; + } else { + this._started = true; + this._schema = schema; + this._writeSchema(schema); + } + } + + return this; + } + + public write(payload?: Table<T> | RecordBatch<T> | Iterable<RecordBatch<T>> | null) { + let schema: Schema<T> | null = null; + + if (!this._sink) { + throw new Error(`RecordBatchWriter is closed`); + } else if (payload == null) { + return this.finish() && undefined; + } else if (payload instanceof Table && !(schema = payload.schema)) { + return this.finish() && undefined; + } else if (payload instanceof RecordBatch && !(schema = payload.schema)) { + return this.finish() && undefined; + } + + if (schema && !compareSchemas(schema, this._schema)) { + if (this._started && this._autoDestroy) { + return this.close(); + } + this.reset(this._sink, schema); + } + + if (payload instanceof RecordBatch) { + if (!(payload instanceof _InternalEmptyPlaceholderRecordBatch)) { + this._writeRecordBatch(payload); + } + } else if (payload instanceof Table) { + this.writeAll(payload.chunks); + } else if (isIterable(payload)) { + this.writeAll(payload); + } + } + + protected _writeMessage<T extends MessageHeader>(message: Message<T>, alignment = 8) { + const a = alignment - 1; + const buffer = Message.encode(message); + const flatbufferSize = buffer.byteLength; + const prefixSize = !this._writeLegacyIpcFormat ? 8 : 4; + const alignedSize = (flatbufferSize + prefixSize + a) & ~a; + const nPaddingBytes = alignedSize - flatbufferSize - prefixSize; + + if (message.headerType === MessageHeader.RecordBatch) { + this._recordBatchBlocks.push(new FileBlock(alignedSize, message.bodyLength, this._position)); + } else if (message.headerType === MessageHeader.DictionaryBatch) { + this._dictionaryBlocks.push(new FileBlock(alignedSize, message.bodyLength, this._position)); + } + + // If not in legacy pre-0.15.0 mode, write the stream continuation indicator + if (!this._writeLegacyIpcFormat) { + this._write(Int32Array.of(-1)); + } + // Write the flatbuffer size prefix including padding + this._write(Int32Array.of(alignedSize - prefixSize)); + // Write the flatbuffer + if (flatbufferSize > 0) { this._write(buffer); } + // Write any padding + return this._writePadding(nPaddingBytes); + } + + protected _write(chunk: ArrayBufferViewInput) { + if (this._started) { + const buffer = toUint8Array(chunk); + if (buffer && buffer.byteLength > 0) { + this._sink.write(buffer); + this._position += buffer.byteLength; + } + } + return this; + } + + protected _writeSchema(schema: Schema<T>) { + return this._writeMessage(Message.from(schema)); + } + + // @ts-ignore + protected _writeFooter(schema: Schema<T>) { + // eos bytes + return this._writeLegacyIpcFormat + ? this._write(Int32Array.of(0)) + : this._write(Int32Array.of(-1, 0)); + } + + protected _writeMagic() { + return this._write(MAGIC); + } + + protected _writePadding(nBytes: number) { + return nBytes > 0 ? this._write(new Uint8Array(nBytes)) : this; + } + + protected _writeRecordBatch(batch: RecordBatch<T>) { + const { byteLength, nodes, bufferRegions, buffers } = VectorAssembler.assemble(batch); + const recordBatch = new metadata.RecordBatch(batch.length, nodes, bufferRegions); + const message = Message.from(recordBatch, byteLength); + return this + ._writeDictionaries(batch) + ._writeMessage(message) + ._writeBodyBuffers(buffers); + } + + protected _writeDictionaryBatch(dictionary: Vector, id: number, isDelta = false) { + this._dictionaryDeltaOffsets.set(id, dictionary.length + (this._dictionaryDeltaOffsets.get(id) || 0)); + const { byteLength, nodes, bufferRegions, buffers } = VectorAssembler.assemble(dictionary); + const recordBatch = new metadata.RecordBatch(dictionary.length, nodes, bufferRegions); + const dictionaryBatch = new metadata.DictionaryBatch(recordBatch, id, isDelta); + const message = Message.from(dictionaryBatch, byteLength); + return this + ._writeMessage(message) + ._writeBodyBuffers(buffers); + } + + protected _writeBodyBuffers(buffers: ArrayBufferView[]) { + let buffer: ArrayBufferView; + let size: number, padding: number; + for (let i = -1, n = buffers.length; ++i < n;) { + if ((buffer = buffers[i]) && (size = buffer.byteLength) > 0) { + this._write(buffer); + if ((padding = ((size + 7) & ~7) - size) > 0) { + this._writePadding(padding); + } + } + } + return this; + } + + protected _writeDictionaries(batch: RecordBatch<T>) { + for (let [id, dictionary] of batch.dictionaries) { + let offset = this._dictionaryDeltaOffsets.get(id) || 0; + if (offset === 0 || (dictionary = dictionary.slice(offset)).length > 0) { + const chunks = 'chunks' in dictionary ? (dictionary as any).chunks : [dictionary]; + for (const chunk of chunks) { + this._writeDictionaryBatch(chunk, id, offset > 0); + offset += chunk.length; + } + } + } + return this; + } +} + +/** @ignore */ +export class RecordBatchStreamWriter<T extends { [key: string]: DataType } = any> extends RecordBatchWriter<T> { + public static writeAll<T extends { [key: string]: DataType } = any>(input: Table<T> | Iterable<RecordBatch<T>>, options?: RecordBatchStreamWriterOptions): RecordBatchStreamWriter<T>; + public static writeAll<T extends { [key: string]: DataType } = any>(input: AsyncIterable<RecordBatch<T>>, options?: RecordBatchStreamWriterOptions): Promise<RecordBatchStreamWriter<T>>; + public static writeAll<T extends { [key: string]: DataType } = any>(input: PromiseLike<AsyncIterable<RecordBatch<T>>>, options?: RecordBatchStreamWriterOptions): Promise<RecordBatchStreamWriter<T>>; + public static writeAll<T extends { [key: string]: DataType } = any>(input: PromiseLike<Table<T> | Iterable<RecordBatch<T>>>, options?: RecordBatchStreamWriterOptions): Promise<RecordBatchStreamWriter<T>>; + /** @nocollapse */ + public static writeAll<T extends { [key: string]: DataType } = any>(input: any, options?: RecordBatchStreamWriterOptions) { + const writer = new RecordBatchStreamWriter<T>(options); + if (isPromise<any>(input)) { + return input.then((x) => writer.writeAll(x)); + } else if (isAsyncIterable<RecordBatch<T>>(input)) { + return writeAllAsync(writer, input); + } + return writeAll(writer, input); + } +} + +/** @ignore */ +export class RecordBatchFileWriter<T extends { [key: string]: DataType } = any> extends RecordBatchWriter<T> { + public static writeAll<T extends { [key: string]: DataType } = any>(input: Table<T> | Iterable<RecordBatch<T>>): RecordBatchFileWriter<T>; + public static writeAll<T extends { [key: string]: DataType } = any>(input: AsyncIterable<RecordBatch<T>>): Promise<RecordBatchFileWriter<T>>; + public static writeAll<T extends { [key: string]: DataType } = any>(input: PromiseLike<AsyncIterable<RecordBatch<T>>>): Promise<RecordBatchFileWriter<T>>; + public static writeAll<T extends { [key: string]: DataType } = any>(input: PromiseLike<Table<T> | Iterable<RecordBatch<T>>>): Promise<RecordBatchFileWriter<T>>; + /** @nocollapse */ + public static writeAll<T extends { [key: string]: DataType } = any>(input: any) { + const writer = new RecordBatchFileWriter<T>(); + if (isPromise<any>(input)) { + return input.then((x) => writer.writeAll(x)); + } else if (isAsyncIterable<RecordBatch<T>>(input)) { + return writeAllAsync(writer, input); + } + return writeAll(writer, input); + } + + constructor() { + super(); + this._autoDestroy = true; + } + + // @ts-ignore + protected _writeSchema(schema: Schema<T>) { + return this._writeMagic()._writePadding(2); + } + + protected _writeFooter(schema: Schema<T>) { + const buffer = Footer.encode(new Footer( + schema, MetadataVersion.V4, + this._recordBatchBlocks, this._dictionaryBlocks + )); + return super + ._writeFooter(schema) // EOS bytes for sequential readers + ._write(buffer) // Write the flatbuffer + ._write(Int32Array.of(buffer.byteLength)) // then the footer size suffix + ._writeMagic(); // then the magic suffix + } +} + +/** @ignore */ +export class RecordBatchJSONWriter<T extends { [key: string]: DataType } = any> extends RecordBatchWriter<T> { + + public static writeAll<T extends { [key: string]: DataType } = any>(this: typeof RecordBatchWriter, input: Table<T> | Iterable<RecordBatch<T>>): RecordBatchJSONWriter<T>; + // @ts-ignore + public static writeAll<T extends { [key: string]: DataType } = any>(this: typeof RecordBatchWriter, input: AsyncIterable<RecordBatch<T>>): Promise<RecordBatchJSONWriter<T>>; + public static writeAll<T extends { [key: string]: DataType } = any>(this: typeof RecordBatchWriter, input: PromiseLike<AsyncIterable<RecordBatch<T>>>): Promise<RecordBatchJSONWriter<T>>; + public static writeAll<T extends { [key: string]: DataType } = any>(this: typeof RecordBatchWriter, input: PromiseLike<Table<T> | Iterable<RecordBatch<T>>>): Promise<RecordBatchJSONWriter<T>>; + /** @nocollapse */ + public static writeAll<T extends { [key: string]: DataType } = any>(this: typeof RecordBatchWriter, input: any) { + return new RecordBatchJSONWriter<T>().writeAll(input as any); + } + + private _recordBatches: RecordBatch[]; + private _dictionaries: RecordBatch[]; + + constructor() { + super(); + this._autoDestroy = true; + this._recordBatches = []; + this._dictionaries = []; + } + + protected _writeMessage() { return this; } + // @ts-ignore + protected _writeFooter(schema: Schema<T>) { return this; } + protected _writeSchema(schema: Schema<T>) { + return this._write(`{\n "schema": ${ + JSON.stringify({ fields: schema.fields.map(fieldToJSON) }, null, 2) + }`); + } + protected _writeDictionaries(batch: RecordBatch<T>) { + if (batch.dictionaries.size > 0) { + this._dictionaries.push(batch); + } + return this; + } + protected _writeDictionaryBatch(dictionary: Vector, id: number, isDelta = false) { + this._dictionaryDeltaOffsets.set(id, dictionary.length + (this._dictionaryDeltaOffsets.get(id) || 0)); + this._write(this._dictionaryBlocks.length === 0 ? ` ` : `,\n `); + this._write(`${dictionaryBatchToJSON(dictionary, id, isDelta)}`); + this._dictionaryBlocks.push(new FileBlock(0, 0, 0)); + return this; + } + protected _writeRecordBatch(batch: RecordBatch<T>) { + this._writeDictionaries(batch); + this._recordBatches.push(batch); + return this; + } + public close() { + + if (this._dictionaries.length > 0) { + this._write(`,\n "dictionaries": [\n`); + for (const batch of this._dictionaries) { + super._writeDictionaries(batch); + } + this._write(`\n ]`); + } + + if (this._recordBatches.length > 0) { + for (let i = -1, n = this._recordBatches.length; ++i < n;) { + this._write(i === 0 ? `,\n "batches": [\n ` : `,\n `); + this._write(`${recordBatchToJSON(this._recordBatches[i])}`); + this._recordBatchBlocks.push(new FileBlock(0, 0, 0)); + } + this._write(`\n ]`); + } + + if (this._schema) { + this._write(`\n}`); + } + + this._dictionaries = []; + this._recordBatches = []; + + return super.close(); + } +} + +/** @ignore */ +function writeAll<T extends { [key: string]: DataType } = any>(writer: RecordBatchWriter<T>, input: Table<T> | Iterable<RecordBatch<T>>) { + let chunks = input as Iterable<RecordBatch<T>>; + if (input instanceof Table) { + chunks = input.chunks; + writer.reset(undefined, input.schema); + } + for (const batch of chunks) { + writer.write(batch); + } + return writer.finish(); +} + +/** @ignore */ +async function writeAllAsync<T extends { [key: string]: DataType } = any>(writer: RecordBatchWriter<T>, batches: AsyncIterable<RecordBatch<T>>) { + for await (const batch of batches) { + writer.write(batch); + } + return writer.finish(); +} + +/** @ignore */ +function fieldToJSON({ name, type, nullable }: Field): Record<string, unknown> { + const assembler = new JSONTypeAssembler(); + return { + 'name': name, 'nullable': nullable, + 'type': assembler.visit(type), + 'children': (type.children || []).map(fieldToJSON), + 'dictionary': !DataType.isDictionary(type) ? undefined : { + 'id': type.id, + 'isOrdered': type.isOrdered, + 'indexType': assembler.visit(type.indices) + } + }; +} + +/** @ignore */ +function dictionaryBatchToJSON(dictionary: Vector, id: number, isDelta = false) { + const field = new Field(`${id}`, dictionary.type, dictionary.nullCount > 0); + const columns = JSONVectorAssembler.assemble(new Column(field, [dictionary])); + return JSON.stringify({ + 'id': id, + 'isDelta': isDelta, + 'data': { + 'count': dictionary.length, + 'columns': columns + } + }, null, 2); +} + +/** @ignore */ +function recordBatchToJSON(records: RecordBatch) { + return JSON.stringify({ + 'count': records.length, + 'columns': JSONVectorAssembler.assemble(records) + }, null, 2); +} diff --git a/src/arrow/js/src/recordbatch.ts b/src/arrow/js/src/recordbatch.ts new file mode 100644 index 000000000..5463a387f --- /dev/null +++ b/src/arrow/js/src/recordbatch.ts @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from './data'; +import { Table } from './table'; +import { Vector } from './vector'; +import { Visitor } from './visitor'; +import { Schema, Field } from './schema'; +import { isIterable } from './util/compat'; +import { Chunked } from './vector/chunked'; +import { selectFieldArgs } from './util/args'; +import { DataType, Struct, Dictionary } from './type'; +import { ensureSameLengthData } from './util/recordbatch'; +import { Clonable, Sliceable, Applicative } from './vector'; +import { StructVector, VectorBuilderOptions, VectorBuilderOptionsAsync } from './vector/index'; + +type VectorMap = { [key: string]: Vector }; +type Fields<T extends { [key: string]: DataType }> = (keyof T)[] | Field<T[keyof T]>[]; +type ChildData<T extends { [key: string]: DataType }> = (Data<T[keyof T]> | Vector<T[keyof T]>)[]; + +export interface RecordBatch<T extends { [key: string]: DataType } = any> { + concat(...others: Vector<Struct<T>>[]): Table<T>; + slice(begin?: number, end?: number): RecordBatch<T>; + clone(data: Data<Struct<T>>, children?: Vector[]): RecordBatch<T>; +} + +export class RecordBatch<T extends { [key: string]: DataType } = any> + extends StructVector<T> + implements Clonable<RecordBatch<T>>, + Sliceable<RecordBatch<T>>, + Applicative<Struct<T>, Table<T>> { + + public static from<T extends { [key: string]: DataType } = any, TNull = any>(options: VectorBuilderOptions<Struct<T>, TNull>): Table<T>; + public static from<T extends { [key: string]: DataType } = any, TNull = any>(options: VectorBuilderOptionsAsync<Struct<T>, TNull>): Promise<Table<T>>; + /** @nocollapse */ + public static from<T extends { [key: string]: DataType } = any, TNull = any>(options: VectorBuilderOptions<Struct<T>, TNull> | VectorBuilderOptionsAsync<Struct<T>, TNull>) { + if (isIterable<(Struct<T>)['TValue'] | TNull>(options['values'])) { + return Table.from(options as VectorBuilderOptions<Struct<T>, TNull>); + } + return Table.from(options as VectorBuilderOptionsAsync<Struct<T>, TNull>); + } + + public static new<T extends VectorMap = any>(children: T): RecordBatch<{ [P in keyof T]: T[P]['type'] }>; + public static new<T extends { [key: string]: DataType } = any>(children: ChildData<T>, fields?: Fields<T>): RecordBatch<T>; + /** @nocollapse */ + public static new<T extends { [key: string]: DataType } = any>(...args: any[]) { + const [fs, xs] = selectFieldArgs<T>(args); + const vs = xs.filter((x): x is Vector<T[keyof T]> => x instanceof Vector); + return new RecordBatch(...ensureSameLengthData(new Schema<T>(fs), vs.map((x) => x.data))); + } + + protected _schema: Schema; + protected _dictionaries?: Map<number, Vector>; + + constructor(schema: Schema<T>, length: number, children: (Data | Vector)[]); + constructor(schema: Schema<T>, data: Data<Struct<T>>, children?: Vector[]); + constructor(...args: any[]) { + let data: Data<Struct<T>>; + const schema = args[0] as Schema<T>; + let children: Vector[] | undefined; + if (args[1] instanceof Data) { + [, data, children] = (args as [any, Data<Struct<T>>, Vector<T[keyof T]>[]?]); + } else { + const fields = schema.fields as Field<T[keyof T]>[]; + const [, length, childData] = args as [any, number, Data<T[keyof T]>[]]; + data = Data.Struct(new Struct<T>(fields), 0, length, 0, null, childData); + } + super(data, children); + this._schema = schema; + } + + public clone(data: Data<Struct<T>>, children = this._children) { + return new RecordBatch<T>(this._schema, data, children); + } + + public concat(...others: Vector<Struct<T>>[]): Table<T> { + const schema = this._schema, chunks = Chunked.flatten(this, ...others); + return new Table(schema, chunks.map(({ data }) => new RecordBatch(schema, data))); + } + + public get schema() { return this._schema; } + public get numCols() { return this._schema.fields.length; } + public get dictionaries() { + return this._dictionaries || (this._dictionaries = DictionaryCollector.collect(this)); + } + + public select<K extends keyof T = any>(...columnNames: K[]) { + const nameToIndex = this._schema.fields.reduce((m, f, i) => m.set(f.name as K, i), new Map<K, number>()); + return this.selectAt(...columnNames.map((columnName) => nameToIndex.get(columnName)!).filter((x) => x > -1)); + } + public selectAt<K extends T[keyof T] = any>(...columnIndices: number[]) { + const schema = this._schema.selectAt(...columnIndices); + const childData = columnIndices.map((i) => this.data.childData[i]).filter(Boolean); + return new RecordBatch<{ [key: string]: K }>(schema, this.length, childData); + } +} + +/** + * An internal class used by the `RecordBatchReader` and `RecordBatchWriter` + * implementations to differentiate between a stream with valid zero-length + * RecordBatches, and a stream with a Schema message, but no RecordBatches. + * @see https://github.com/apache/arrow/pull/4373 + * @ignore + * @private + */ +/* eslint-disable @typescript-eslint/naming-convention */ +export class _InternalEmptyPlaceholderRecordBatch<T extends { [key: string]: DataType } = any> extends RecordBatch<T> { + constructor(schema: Schema<T>) { + super(schema, 0, schema.fields.map((f) => Data.new(f.type, 0, 0, 0))); + } +} + +/** @ignore */ +class DictionaryCollector extends Visitor { + public dictionaries = new Map<number, Vector>(); + public static collect<T extends RecordBatch>(batch: T) { + return new DictionaryCollector().visit( + batch.data, new Struct(batch.schema.fields) + ).dictionaries; + } + public visit(data: Data, type: DataType) { + if (DataType.isDictionary(type)) { + return this.visitDictionary(data, type); + } else { + data.childData.forEach((child, i) => + this.visit(child, type.children[i].type)); + } + return this; + } + public visitDictionary(data: Data, type: Dictionary) { + const dictionary = data.dictionary; + if (dictionary && dictionary.length > 0) { + this.dictionaries.set(type.id, dictionary); + } + return this; + } +} diff --git a/src/arrow/js/src/schema.ts b/src/arrow/js/src/schema.ts new file mode 100644 index 000000000..437ffa228 --- /dev/null +++ b/src/arrow/js/src/schema.ts @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { DataType } from './type'; + +export class Schema<T extends { [key: string]: DataType } = any> { + + public readonly fields: Field<T[keyof T]>[]; + public readonly metadata: Map<string, string>; + public readonly dictionaries: Map<number, DataType>; + + constructor(fields: Field[] = [], + metadata?: Map<string, string> | null, + dictionaries?: Map<number, DataType> | null) { + this.fields = (fields || []) as Field<T[keyof T]>[]; + this.metadata = metadata || new Map(); + if (!dictionaries) { + dictionaries = generateDictionaryMap(fields); + } + this.dictionaries = dictionaries; + } + public get [Symbol.toStringTag]() { return 'Schema'; } + public toString() { + return `Schema<{ ${this.fields.map((f, i) => `${i}: ${f}`).join(', ')} }>`; + } + + public select<K extends keyof T = any>(...columnNames: K[]) { + const names = columnNames.reduce((xs, x) => (xs[x] = true) && xs, Object.create(null)); + return new Schema<{ [P in K]: T[P] }>(this.fields.filter((f) => names[f.name]), this.metadata); + } + public selectAt<K extends T[keyof T] = any>(...columnIndices: number[]) { + return new Schema<{ [key: string]: K }>(columnIndices.map((i) => this.fields[i]).filter(Boolean), this.metadata); + } + + public assign<R extends { [key: string]: DataType } = any>(schema: Schema<R>): Schema<T & R>; + public assign<R extends { [key: string]: DataType } = any>(...fields: (Field<R[keyof R]> | Field<R[keyof R]>[])[]): Schema<T & R>; + public assign<R extends { [key: string]: DataType } = any>(...args: (Schema<R> | Field<R[keyof R]> | Field<R[keyof R]>[])[]) { + + const other = (args[0] instanceof Schema + ? args[0] as Schema<R> + : Array.isArray(args[0]) + ? new Schema<R>(<Field<R[keyof R]>[]> args[0]) + : new Schema<R>(<Field<R[keyof R]>[]> args)); + + const curFields = [...this.fields] as Field[]; + const metadata = mergeMaps(mergeMaps(new Map(), this.metadata), other.metadata); + const newFields = other.fields.filter((f2) => { + const i = curFields.findIndex((f) => f.name === f2.name); + return ~i ? (curFields[i] = f2.clone({ + metadata: mergeMaps(mergeMaps(new Map(), curFields[i].metadata), f2.metadata) + })) && false : true; + }) as Field[]; + + const newDictionaries = generateDictionaryMap(newFields, new Map()); + + return new Schema<T & R>( + [...curFields, ...newFields], metadata, + new Map([...this.dictionaries, ...newDictionaries]) + ); + } +} + +export class Field<T extends DataType = any> { + + public static new<T extends DataType = any>(props: { name: string | number; type: T; nullable?: boolean; metadata?: Map<string, string> | null }): Field<T>; + public static new<T extends DataType = any>(name: string | number | Field<T>, type: T, nullable?: boolean, metadata?: Map<string, string> | null): Field<T>; + /** @nocollapse */ + public static new<T extends DataType = any>(...args: any[]) { + let [name, type, nullable, metadata] = args; + if (args[0] && typeof args[0] === 'object') { + ({ name } = args[0]); + (type === undefined) && (type = args[0].type); + (nullable === undefined) && (nullable = args[0].nullable); + (metadata === undefined) && (metadata = args[0].metadata); + } + return new Field<T>(`${name}`, type, nullable, metadata); + } + + public readonly type: T; + public readonly name: string; + public readonly nullable: boolean; + public readonly metadata: Map<string, string>; + + constructor(name: string, type: T, nullable = false, metadata?: Map<string, string> | null) { + this.name = name; + this.type = type; + this.nullable = nullable; + this.metadata = metadata || new Map(); + } + + public get typeId() { return this.type.typeId; } + public get [Symbol.toStringTag]() { return 'Field'; } + public toString() { return `${this.name}: ${this.type}`; } + public clone<R extends DataType = T>(props: { name?: string | number; type?: R; nullable?: boolean; metadata?: Map<string, string> | null }): Field<R>; + public clone<R extends DataType = T>(name?: string | number | Field<T>, type?: R, nullable?: boolean, metadata?: Map<string, string> | null): Field<R>; + public clone<R extends DataType = T>(...args: any[]) { + let [name, type, nullable, metadata] = args; + (!args[0] || typeof args[0] !== 'object') + ? ([name = this.name, type = this.type, nullable = this.nullable, metadata = this.metadata] = args) + : ({name = this.name, type = this.type, nullable = this.nullable, metadata = this.metadata} = args[0]); + return Field.new<R>(name, type, nullable, metadata); + } +} + +/** @ignore */ +function mergeMaps<TKey, TVal>(m1?: Map<TKey, TVal> | null, m2?: Map<TKey, TVal> | null): Map<TKey, TVal> { + return new Map([...(m1 || new Map()), ...(m2 || new Map())]); +} + +/** @ignore */ +function generateDictionaryMap(fields: Field[], dictionaries = new Map<number, DataType>()): Map<number, DataType> { + + for (let i = -1, n = fields.length; ++i < n;) { + const field = fields[i]; + const type = field.type; + if (DataType.isDictionary(type)) { + if (!dictionaries.has(type.id)) { + dictionaries.set(type.id, type.dictionary); + } else if (dictionaries.get(type.id) !== type.dictionary) { + throw new Error(`Cannot create Schema containing two different dictionaries with the same Id`); + } + } + if (type.children && type.children.length > 0) { + generateDictionaryMap(type.children, dictionaries); + } + } + + return dictionaries; +} + +// Add these here so they're picked up by the externs creator +// in the build, and closure-compiler doesn't minify them away +(Schema.prototype as any).fields = null; +(Schema.prototype as any).metadata = null; +(Schema.prototype as any).dictionaries = null; + +(Field.prototype as any).type = null; +(Field.prototype as any).name = null; +(Field.prototype as any).nullable = null; +(Field.prototype as any).metadata = null; diff --git a/src/arrow/js/src/table.ts b/src/arrow/js/src/table.ts new file mode 100644 index 000000000..d5e121de7 --- /dev/null +++ b/src/arrow/js/src/table.ts @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Column } from './column'; +import { Data } from './data'; +import { TypedArray, TypedArrayDataType } from './interfaces'; +import { RecordBatchReader } from './ipc/reader'; +import { RecordBatchFileWriter, RecordBatchStreamWriter } from './ipc/writer'; +import { RecordBatch, _InternalEmptyPlaceholderRecordBatch } from './recordbatch'; +import { Field, Schema } from './schema'; +import { DataType, RowLike, Struct } from './type'; +import { selectArgs, selectColumnArgs } from './util/args'; +import { isAsyncIterable, isIterable, isPromise } from './util/compat'; +import { distributeColumnsIntoRecordBatches, distributeVectorsIntoRecordBatches } from './util/recordbatch'; +import { Applicative, Clonable, Sliceable } from './vector'; +import { Chunked, StructVector, Vector, VectorBuilderOptions, VectorBuilderOptionsAsync } from './vector/index'; + +type VectorMap = { [key: string]: Vector | Exclude<TypedArray, Uint8ClampedArray> }; +type Fields<T extends { [key: string]: DataType }> = (keyof T)[] | Field<T[keyof T]>[]; +type ChildData<T extends { [key: string]: DataType }> = Data<T[keyof T]>[] | Vector<T[keyof T]>[]; +type Columns<T extends { [key: string]: DataType }> = Column<T[keyof T]>[] | Column<T[keyof T]>[][]; + +export interface Table<T extends { [key: string]: DataType } = any> { + + get(index: number): Struct<T>['TValue']; + [Symbol.iterator](): IterableIterator<RowLike<T>>; + + slice(begin?: number, end?: number): Table<T>; + concat(...others: Vector<Struct<T>>[]): Table<T>; + clone(chunks?: RecordBatch<T>[], offsets?: Uint32Array): Table<T>; +} + +export class Table<T extends { [key: string]: DataType } = any> + extends Chunked<Struct<T>> + implements Clonable<Table<T>>, + Sliceable<Table<T>>, + Applicative<Struct<T>, Table<T>> { + + /** @nocollapse */ + public static empty<T extends { [key: string]: DataType } = Record<string, never>>(schema = new Schema<T>([])) { return new Table<T>(schema, []); } + + public static from(): Table<Record<string, never>>; + public static from<T extends { [key: string]: DataType } = any>(source: RecordBatchReader<T>): Table<T>; + public static from<T extends { [key: string]: DataType } = any>(source: import('./ipc/reader').FromArg0): Table<T>; + public static from<T extends { [key: string]: DataType } = any>(source: import('./ipc/reader').FromArg2): Table<T>; + public static from<T extends { [key: string]: DataType } = any>(source: import('./ipc/reader').FromArg1): Promise<Table<T>>; + public static from<T extends { [key: string]: DataType } = any>(source: import('./ipc/reader').FromArg3): Promise<Table<T>>; + public static from<T extends { [key: string]: DataType } = any>(source: import('./ipc/reader').FromArg4): Promise<Table<T>>; + public static from<T extends { [key: string]: DataType } = any>(source: import('./ipc/reader').FromArg5): Promise<Table<T>>; + public static from<T extends { [key: string]: DataType } = any>(source: PromiseLike<RecordBatchReader<T>>): Promise<Table<T>>; + public static from<T extends { [key: string]: DataType } = any, TNull = any>(options: VectorBuilderOptions<Struct<T>, TNull>): Table<T>; + public static from<T extends { [key: string]: DataType } = any, TNull = any>(options: VectorBuilderOptionsAsync<Struct<T>, TNull>): Promise<Table<T>>; + /** @nocollapse */ + public static from<T extends { [key: string]: DataType } = any, TNull = any>(input?: any) { + + if (!input) { return Table.empty(); } + + if (typeof input === 'object') { + const table = isIterable(input['values']) ? tableFromIterable<T, TNull>(input) + : isAsyncIterable(input['values']) ? tableFromAsyncIterable<T, TNull>(input) + : null; + if (table !== null) { return table; } + } + + let reader = RecordBatchReader.from<T>(input) as RecordBatchReader<T> | Promise<RecordBatchReader<T>>; + + if (isPromise<RecordBatchReader<T>>(reader)) { + return (async () => await Table.from(await reader))(); + } + if (reader.isSync() && (reader = reader.open())) { + return !reader.schema ? Table.empty() : new Table<T>(reader.schema, [...reader]); + } + return (async (opening) => { + const reader = await opening; + const schema = reader.schema; + const batches: RecordBatch[] = []; + if (schema) { + for await (const batch of reader) { + batches.push(batch); + } + return new Table<T>(schema, batches); + } + return Table.empty(); + })(reader.open()); + } + + /** @nocollapse */ + public static async fromAsync<T extends { [key: string]: DataType } = any>(source: import('./ipc/reader').FromArgs): Promise<Table<T>> { + return await Table.from<T>(source as any); + } + + /** @nocollapse */ + public static fromStruct<T extends { [key: string]: DataType } = any>(vector: Vector<Struct<T>>) { + return Table.new<T>(vector.data.childData as Data<T[keyof T]>[], vector.type.children); + } + + /** + * @summary Create a new Table from a collection of Columns or Vectors, + * with an optional list of names or Fields. + * + * + * `Table.new` accepts an Object of + * Columns or Vectors, where the keys will be used as the field names + * for the Schema: + * ```ts + * const i32s = Int32Vector.from([1, 2, 3]); + * const f32s = Float32Vector.from([.1, .2, .3]); + * const table = Table.new({ i32: i32s, f32: f32s }); + * assert(table.schema.fields[0].name === 'i32'); + * ``` + * + * It also accepts a a list of Vectors with an optional list of names or + * Fields for the resulting Schema. If the list is omitted or a name is + * missing, the numeric index of each Vector will be used as the name: + * ```ts + * const i32s = Int32Vector.from([1, 2, 3]); + * const f32s = Float32Vector.from([.1, .2, .3]); + * const table = Table.new([i32s, f32s], ['i32']); + * assert(table.schema.fields[0].name === 'i32'); + * assert(table.schema.fields[1].name === '1'); + * ``` + * + * If the supplied arguments are Columns, `Table.new` will infer the Schema + * from the Columns: + * ```ts + * const i32s = Column.new('i32', Int32Vector.from([1, 2, 3])); + * const f32s = Column.new('f32', Float32Vector.from([.1, .2, .3])); + * const table = Table.new(i32s, f32s); + * assert(table.schema.fields[0].name === 'i32'); + * assert(table.schema.fields[1].name === 'f32'); + * ``` + * + * If the supplied Vector or Column lengths are unequal, `Table.new` will + * extend the lengths of the shorter Columns, allocating additional bytes + * to represent the additional null slots. The memory required to allocate + * these additional bitmaps can be computed as: + * ```ts + * let additionalBytes = 0; + * for (let vec in shorter_vectors) { + * additionalBytes += (((longestLength - vec.length) + 63) & ~63) >> 3; + * } + * ``` + * + * For example, an additional null bitmap for one million null values would require + * 125,000 bytes (`((1e6 + 63) & ~63) >> 3`), or approx. `0.11MiB` + */ + public static new<T extends { [key: string]: DataType } = any>(...columns: Columns<T>): Table<T>; + public static new<T extends VectorMap = any>(children: T): Table<{ [P in keyof T]: T[P] extends Vector ? T[P]['type'] : T[P] extends Exclude<TypedArray, Uint8ClampedArray> ? TypedArrayDataType<T[P]> : never}>; + public static new<T extends { [key: string]: DataType } = any>(children: ChildData<T>, fields?: Fields<T>): Table<T>; + /** @nocollapse */ + public static new(...cols: any[]) { + return new Table(...distributeColumnsIntoRecordBatches(selectColumnArgs(cols))); + } + + constructor(table: Table<T>); + constructor(batches: RecordBatch<T>[]); + constructor(...batches: RecordBatch<T>[]); + constructor(schema: Schema<T>, batches: RecordBatch<T>[]); + constructor(schema: Schema<T>, ...batches: RecordBatch<T>[]); + constructor(...args: any[]) { + + let schema: Schema<T> = null!; + + if (args[0] instanceof Schema) { schema = args[0]; } + + const chunks = args[0] instanceof Table ? (args[0] as Table<T>).chunks : selectArgs<RecordBatch<T>>(RecordBatch, args); + + if (!schema && !(schema = chunks[0]?.schema)) { + throw new TypeError('Table must be initialized with a Schema or at least one RecordBatch'); + } + + chunks[0] || (chunks[0] = new _InternalEmptyPlaceholderRecordBatch(schema)); + + super(new Struct(schema.fields), chunks); + + this._schema = schema; + this._chunks = chunks; + } + + protected _schema: Schema<T>; + // List of inner RecordBatches + protected _chunks: RecordBatch<T>[]; + protected _children?: Column<T[keyof T]>[]; + + public get schema() { return this._schema; } + public get length() { return this._length; } + public get chunks() { return this._chunks; } + public get numCols() { return this._numChildren; } + + public clone(chunks = this._chunks) { + return new Table<T>(this._schema, chunks); + } + + public getColumn<R extends keyof T>(name: R): Column<T[R]> { + return this.getColumnAt(this.getColumnIndex(name)) as Column<T[R]>; + } + public getColumnAt<R extends DataType = any>(index: number): Column<R> | null { + return this.getChildAt(index); + } + public getColumnIndex<R extends keyof T>(name: R) { + return this._schema.fields.findIndex((f) => f.name === name); + } + public getChildAt<R extends DataType = any>(index: number): Column<R> | null { + if (index < 0 || index >= this.numChildren) { return null; } + let field: Field<R>, child: Column<R>; + const fields = (this._schema as Schema<any>).fields; + const columns = this._children || (this._children = []) as Column[]; + if (child = columns[index]) { return child as Column<R>; } + if (field = fields[index]) { + const chunks = this._chunks + .map((chunk) => chunk.getChildAt<R>(index)) + .filter((vec): vec is Vector<R> => vec != null); + if (chunks.length > 0) { + return (columns[index] = new Column<R>(field, chunks)); + } + } + return null; + } + + // @ts-ignore + public serialize(encoding = 'binary', stream = true) { + const Writer = !stream + ? RecordBatchFileWriter + : RecordBatchStreamWriter; + return Writer.writeAll(this).toUint8Array(true); + } + public count(): number { + return this._length; + } + public select<K extends keyof T = any>(...columnNames: K[]) { + const nameToIndex = this._schema.fields.reduce((m, f, i) => m.set(f.name as K, i), new Map<K, number>()); + return this.selectAt(...columnNames.map((columnName) => nameToIndex.get(columnName)!).filter((x) => x > -1)); + } + public selectAt<K extends T[keyof T] = any>(...columnIndices: number[]) { + const schema = this._schema.selectAt<K>(...columnIndices); + return new Table(schema, this._chunks.map(({ length, data: { childData } }) => { + return new RecordBatch(schema, length, columnIndices.map((i) => childData[i]).filter(Boolean)); + })); + } + public assign<R extends { [key: string]: DataType } = any>(other: Table<R>) { + + const fields = this._schema.fields; + const [indices, oldToNew] = other.schema.fields.reduce((memo, f2, newIdx) => { + const [indices, oldToNew] = memo; + const i = fields.findIndex((f) => f.name === f2.name); + ~i ? (oldToNew[i] = newIdx) : indices.push(newIdx); + return memo; + }, [[], []] as number[][]); + + const schema = this._schema.assign(other.schema); + const columns = [ + ...fields.map((_f, i, _fs, j = oldToNew[i]) => + (j === undefined ? this.getColumnAt(i) : other.getColumnAt(j))!), + ...indices.map((i) => other.getColumnAt(i)!) + ].filter(Boolean) as Column<(T & R)[keyof T | keyof R]>[]; + + return new Table<T & R>(...distributeVectorsIntoRecordBatches<any>(schema, columns)); + } +} + +function tableFromIterable<T extends { [key: string]: DataType } = any, TNull = any>(input: VectorBuilderOptions<Struct<T>, TNull>) { + const { type } = input; + if (type instanceof Struct) { + return Table.fromStruct(StructVector.from(input as VectorBuilderOptions<Struct<T>, TNull>)); + } + return null; +} + +function tableFromAsyncIterable<T extends { [key: string]: DataType } = any, TNull = any>(input: VectorBuilderOptionsAsync<Struct<T>, TNull>) { + const { type } = input; + if (type instanceof Struct) { + return StructVector.from(input as VectorBuilderOptionsAsync<Struct<T>, TNull>).then((vector) => Table.fromStruct(vector)); + } + return null; +} diff --git a/src/arrow/js/src/type.ts b/src/arrow/js/src/type.ts new file mode 100644 index 000000000..7d5c051ad --- /dev/null +++ b/src/arrow/js/src/type.ts @@ -0,0 +1,613 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* eslint-disable @typescript-eslint/naming-convention */ + +import { Field } from './schema'; +import { flatbuffers } from 'flatbuffers'; +import { VectorType as V } from './interfaces'; +import { TypedArrayConstructor } from './interfaces'; + +import Long = flatbuffers.Long; +import { + Type, + Precision, UnionMode, + DateUnit, TimeUnit, IntervalUnit +} from './enum'; + +/** @ignore */ +export type TimeBitWidth = 32 | 64; +/** @ignore */ +export type IntBitWidth = 8 | 16 | 32 | 64; +/** @ignore */ +export type IsSigned = { 'true': true; 'false': false }; +/** @ignore */ +export type RowLike<T extends { [key: string]: DataType }> = + ( Iterable<[string, T[keyof T]['TValue'] | null]> ) + & { [P in keyof T]: T[P]['TValue'] | null } + & { get<K extends keyof T>(key: K): T[K]['TValue'] | null } + & { set<K extends keyof T>(key: K, val: T[K]['TValue'] | null): void } + ; + +/** @ignore */ +export type MapLike<K extends DataType = any, V extends DataType = any> = + { [P in K['TValue']]: V['TValue'] | null } + & ( Map<K['TValue'], V['TValue'] | null> ) + ; + +export interface DataType<TType extends Type = Type, TChildren extends { [key: string]: DataType } = any> { + readonly TType: TType; + readonly TArray: any; + readonly TValue: any; + readonly ArrayType: any; + readonly children: Field<TChildren[keyof TChildren]>[]; +} + +/** + * An abstract base class for classes that encapsulate metadata about each of + * the logical types that Arrow can represent. + */ +export abstract class DataType<TType extends Type = Type, TChildren extends { [key: string]: DataType } = any> { + + public [Symbol.toStringTag]: string; + + /** @nocollapse */ static isNull (x: any): x is Null { return x?.typeId === Type.Null; } + /** @nocollapse */ static isInt (x: any): x is Int_ { return x?.typeId === Type.Int; } + /** @nocollapse */ static isFloat (x: any): x is Float { return x?.typeId === Type.Float; } + /** @nocollapse */ static isBinary (x: any): x is Binary { return x?.typeId === Type.Binary; } + /** @nocollapse */ static isUtf8 (x: any): x is Utf8 { return x?.typeId === Type.Utf8; } + /** @nocollapse */ static isBool (x: any): x is Bool { return x?.typeId === Type.Bool; } + /** @nocollapse */ static isDecimal (x: any): x is Decimal { return x?.typeId === Type.Decimal; } + /** @nocollapse */ static isDate (x: any): x is Date_ { return x?.typeId === Type.Date; } + /** @nocollapse */ static isTime (x: any): x is Time_ { return x?.typeId === Type.Time; } + /** @nocollapse */ static isTimestamp (x: any): x is Timestamp_ { return x?.typeId === Type.Timestamp; } + /** @nocollapse */ static isInterval (x: any): x is Interval_ { return x?.typeId === Type.Interval; } + /** @nocollapse */ static isList (x: any): x is List { return x?.typeId === Type.List; } + /** @nocollapse */ static isStruct (x: any): x is Struct { return x?.typeId === Type.Struct; } + /** @nocollapse */ static isUnion (x: any): x is Union_ { return x?.typeId === Type.Union; } + /** @nocollapse */ static isFixedSizeBinary (x: any): x is FixedSizeBinary { return x?.typeId === Type.FixedSizeBinary; } + /** @nocollapse */ static isFixedSizeList (x: any): x is FixedSizeList { return x?.typeId === Type.FixedSizeList; } + /** @nocollapse */ static isMap (x: any): x is Map_ { return x?.typeId === Type.Map; } + /** @nocollapse */ static isDictionary (x: any): x is Dictionary { return x?.typeId === Type.Dictionary; } + + public get typeId(): TType { return <any> Type.NONE; } + + protected static [Symbol.toStringTag] = ((proto: DataType) => { + (<any> proto).children = null; + (<any> proto).ArrayType = Array; + return proto[Symbol.toStringTag] = 'DataType'; + })(DataType.prototype); +} + +/** @ignore */ +export interface Null extends DataType<Type.Null> { TArray: void; TValue: null } +/** @ignore */ +export class Null extends DataType<Type.Null> { + public toString() { return `Null`; } + public get typeId() { return Type.Null as Type.Null; } + protected static [Symbol.toStringTag] = ((proto: Null) => { + return proto[Symbol.toStringTag] = 'Null'; + })(Null.prototype); +} + +/** @ignore */ +type Ints = Type.Int | Type.Int8 | Type.Int16 | Type.Int32 | Type.Int64 | Type.Uint8 | Type.Uint16 | Type.Uint32 | Type.Uint64; +/** @ignore */ +type IType = { + [Type.Int ]: { bitWidth: IntBitWidth; isSigned: true | false; TArray: IntArray; TValue: number | bigint | Int32Array | Uint32Array }; + [Type.Int8 ]: { bitWidth: 8; isSigned: true; TArray: Int8Array; TValue: number }; + [Type.Int16 ]: { bitWidth: 16; isSigned: true; TArray: Int16Array; TValue: number }; + [Type.Int32 ]: { bitWidth: 32; isSigned: true; TArray: Int32Array; TValue: number }; + [Type.Int64 ]: { bitWidth: 64; isSigned: true; TArray: Int32Array; TValue: bigint | Int32Array | Uint32Array }; + [Type.Uint8 ]: { bitWidth: 8; isSigned: false; TArray: Uint8Array; TValue: number }; + [Type.Uint16]: { bitWidth: 16; isSigned: false; TArray: Uint16Array; TValue: number }; + [Type.Uint32]: { bitWidth: 32; isSigned: false; TArray: Uint32Array; TValue: number }; + [Type.Uint64]: { bitWidth: 64; isSigned: false; TArray: Uint32Array; TValue: bigint | Int32Array | Uint32Array }; +}; + +/** @ignore */ +interface Int_<T extends Ints = Ints> extends DataType<T> { TArray: IType[T]['TArray']; TValue: IType[T]['TValue'] } +/** @ignore */ +class Int_<T extends Ints = Ints> extends DataType<T> { + constructor(public readonly isSigned: IType[T]['isSigned'], + public readonly bitWidth: IType[T]['bitWidth']) { + super(); + } + public get typeId() { return Type.Int as T; } + public get ArrayType(): TypedArrayConstructor<IType[T]['TArray']> { + switch (this.bitWidth) { + case 8: return this.isSigned ? Int8Array : Uint8Array; + case 16: return this.isSigned ? Int16Array : Uint16Array; + case 32: return this.isSigned ? Int32Array : Uint32Array; + case 64: return this.isSigned ? Int32Array : Uint32Array; + } + throw new Error(`Unrecognized ${this[Symbol.toStringTag]} type`); + } + public toString() { return `${this.isSigned ? `I` : `Ui`}nt${this.bitWidth}`; } + protected static [Symbol.toStringTag] = ((proto: Int_) => { + (<any> proto).isSigned = null; + (<any> proto).bitWidth = null; + return proto[Symbol.toStringTag] = 'Int'; + })(Int_.prototype); +} + +export { Int_ as Int }; + +/** @ignore */ +export class Int8 extends Int_<Type.Int8> { constructor() { super(true, 8); } } +/** @ignore */ +export class Int16 extends Int_<Type.Int16> { constructor() { super(true, 16); } } +/** @ignore */ +export class Int32 extends Int_<Type.Int32> { constructor() { super(true, 32); } } +/** @ignore */ +export class Int64 extends Int_<Type.Int64> { constructor() { super(true, 64); } } +/** @ignore */ +export class Uint8 extends Int_<Type.Uint8> { constructor() { super(false, 8); } } +/** @ignore */ +export class Uint16 extends Int_<Type.Uint16> { constructor() { super(false, 16); } } +/** @ignore */ +export class Uint32 extends Int_<Type.Uint32> { constructor() { super(false, 32); } } +/** @ignore */ +export class Uint64 extends Int_<Type.Uint64> { constructor() { super(false, 64); } } + +Object.defineProperty(Int8.prototype, 'ArrayType', { value: Int8Array }); +Object.defineProperty(Int16.prototype, 'ArrayType', { value: Int16Array }); +Object.defineProperty(Int32.prototype, 'ArrayType', { value: Int32Array }); +Object.defineProperty(Int64.prototype, 'ArrayType', { value: Int32Array }); +Object.defineProperty(Uint8.prototype, 'ArrayType', { value: Uint8Array }); +Object.defineProperty(Uint16.prototype, 'ArrayType', { value: Uint16Array }); +Object.defineProperty(Uint32.prototype, 'ArrayType', { value: Uint32Array }); +Object.defineProperty(Uint64.prototype, 'ArrayType', { value: Uint32Array }); + +/** @ignore */ +type Floats = Type.Float | Type.Float16 | Type.Float32 | Type.Float64; +/** @ignore */ +type FType = { + [Type.Float ]: { precision: Precision; TArray: FloatArray; TValue: number }; + [Type.Float16]: { precision: Precision.HALF; TArray: Uint16Array; TValue: number }; + [Type.Float32]: { precision: Precision.SINGLE; TArray: Float32Array; TValue: number }; + [Type.Float64]: { precision: Precision.DOUBLE; TArray: Float64Array; TValue: number }; +}; + +/** @ignore */ +export interface Float<T extends Floats = Floats> extends DataType<T> { TArray: FType[T]['TArray']; TValue: number } +/** @ignore */ +export class Float<T extends Floats = Floats> extends DataType<T> { + constructor(public readonly precision: Precision) { + super(); + } + public get typeId() { return Type.Float as T; } + public get ArrayType(): TypedArrayConstructor<FType[T]['TArray']> { + switch (this.precision) { + case Precision.HALF: return Uint16Array; + case Precision.SINGLE: return Float32Array; + case Precision.DOUBLE: return Float64Array; + } + // @ts-ignore + throw new Error(`Unrecognized ${this[Symbol.toStringTag]} type`); + } + public toString() { return `Float${(this.precision << 5) || 16}`; } + protected static [Symbol.toStringTag] = ((proto: Float) => { + (<any> proto).precision = null; + return proto[Symbol.toStringTag] = 'Float'; + })(Float.prototype); +} + +/** @ignore */ +export class Float16 extends Float<Type.Float16> { constructor() { super(Precision.HALF); } } +/** @ignore */ +export class Float32 extends Float<Type.Float32> { constructor() { super(Precision.SINGLE); } } +/** @ignore */ +export class Float64 extends Float<Type.Float64> { constructor() { super(Precision.DOUBLE); } } + +Object.defineProperty(Float16.prototype, 'ArrayType', { value: Uint16Array }); +Object.defineProperty(Float32.prototype, 'ArrayType', { value: Float32Array }); +Object.defineProperty(Float64.prototype, 'ArrayType', { value: Float64Array }); + +/** @ignore */ +export interface Binary extends DataType<Type.Binary> { TArray: Uint8Array; TValue: Uint8Array; ArrayType: TypedArrayConstructor<Uint8Array> } +/** @ignore */ +export class Binary extends DataType<Type.Binary> { + constructor() { + super(); + } + public get typeId() { return Type.Binary as Type.Binary; } + public toString() { return `Binary`; } + protected static [Symbol.toStringTag] = ((proto: Binary) => { + (<any> proto).ArrayType = Uint8Array; + return proto[Symbol.toStringTag] = 'Binary'; + })(Binary.prototype); +} + +/** @ignore */ +export interface Utf8 extends DataType<Type.Utf8> { TArray: Uint8Array; TValue: string; ArrayType: TypedArrayConstructor<Uint8Array> } +/** @ignore */ +export class Utf8 extends DataType<Type.Utf8> { + constructor() { + super(); + } + public get typeId() { return Type.Utf8 as Type.Utf8; } + public toString() { return `Utf8`; } + protected static [Symbol.toStringTag] = ((proto: Utf8) => { + (<any> proto).ArrayType = Uint8Array; + return proto[Symbol.toStringTag] = 'Utf8'; + })(Utf8.prototype); +} + +/** @ignore */ +export interface Bool extends DataType<Type.Bool> { TArray: Uint8Array; TValue: boolean; ArrayType: TypedArrayConstructor<Uint8Array> } +/** @ignore */ +export class Bool extends DataType<Type.Bool> { + constructor() { + super(); + } + public get typeId() { return Type.Bool as Type.Bool; } + public toString() { return `Bool`; } + protected static [Symbol.toStringTag] = ((proto: Bool) => { + (<any> proto).ArrayType = Uint8Array; + return proto[Symbol.toStringTag] = 'Bool'; + })(Bool.prototype); +} + +/** @ignore */ +export interface Decimal extends DataType<Type.Decimal> { TArray: Uint32Array; TValue: Uint32Array; ArrayType: TypedArrayConstructor<Uint32Array> } +/** @ignore */ +export class Decimal extends DataType<Type.Decimal> { + constructor(public readonly scale: number, + public readonly precision: number) { + super(); + } + public get typeId() { return Type.Decimal as Type.Decimal; } + public toString() { return `Decimal[${this.precision}e${this.scale > 0 ? `+` : ``}${this.scale}]`; } + protected static [Symbol.toStringTag] = ((proto: Decimal) => { + (<any> proto).scale = null; + (<any> proto).precision = null; + (<any> proto).ArrayType = Uint32Array; + return proto[Symbol.toStringTag] = 'Decimal'; + })(Decimal.prototype); +} + +/** @ignore */ +export type Dates = Type.Date | Type.DateDay | Type.DateMillisecond; +/** @ignore */ +export interface Date_<T extends Dates = Dates> extends DataType<T> { TArray: Int32Array; TValue: Date; ArrayType: TypedArrayConstructor<Int32Array> } +/** @ignore */ +export class Date_<T extends Dates = Dates> extends DataType<T> { + constructor(public readonly unit: DateUnit) { + super(); + } + public get typeId() { return Type.Date as T; } + public toString() { return `Date${(this.unit + 1) * 32}<${DateUnit[this.unit]}>`; } + protected static [Symbol.toStringTag] = ((proto: Date_) => { + (<any> proto).unit = null; + (<any> proto).ArrayType = Int32Array; + return proto[Symbol.toStringTag] = 'Date'; + })(Date_.prototype); +} + +/** @ignore */ +export class DateDay extends Date_<Type.DateDay> { constructor() { super(DateUnit.DAY); } } +/** @ignore */ +export class DateMillisecond extends Date_<Type.DateMillisecond> { constructor() { super(DateUnit.MILLISECOND); } } + +/** @ignore */ +type Times = Type.Time | Type.TimeSecond | Type.TimeMillisecond | Type.TimeMicrosecond | Type.TimeNanosecond; +/** @ignore */ +type TimesType = { + [Type.Time ]: { unit: TimeUnit; TValue: number | Int32Array }; + [Type.TimeSecond ]: { unit: TimeUnit.SECOND; TValue: number }; + [Type.TimeMillisecond]: { unit: TimeUnit.MILLISECOND; TValue: number }; + [Type.TimeMicrosecond]: { unit: TimeUnit.MICROSECOND; TValue: Int32Array }; + [Type.TimeNanosecond ]: { unit: TimeUnit.NANOSECOND; TValue: Int32Array }; +}; + +/** @ignore */ +interface Time_<T extends Times = Times> extends DataType<T> { TArray: Int32Array; TValue: TimesType[T]['TValue']; ArrayType: TypedArrayConstructor<Int32Array> } +/** @ignore */ +class Time_<T extends Times = Times> extends DataType<T> { + constructor(public readonly unit: TimesType[T]['unit'], + public readonly bitWidth: TimeBitWidth) { + super(); + } + public get typeId() { return Type.Time as T; } + public toString() { return `Time${this.bitWidth}<${TimeUnit[this.unit]}>`; } + protected static [Symbol.toStringTag] = ((proto: Time_) => { + (<any> proto).unit = null; + (<any> proto).bitWidth = null; + (<any> proto).ArrayType = Int32Array; + return proto[Symbol.toStringTag] = 'Time'; + })(Time_.prototype); +} + +export { Time_ as Time }; + +/** @ignore */ +export class TimeSecond extends Time_<Type.TimeSecond> { constructor() { super(TimeUnit.SECOND, 32); } } +/** @ignore */ +export class TimeMillisecond extends Time_<Type.TimeMillisecond> { constructor() { super(TimeUnit.MILLISECOND, 32); } } +/** @ignore */ +export class TimeMicrosecond extends Time_<Type.TimeMicrosecond> { constructor() { super(TimeUnit.MICROSECOND, 64); } } +/** @ignore */ +export class TimeNanosecond extends Time_<Type.TimeNanosecond> { constructor() { super(TimeUnit.NANOSECOND, 64); } } + +/** @ignore */ +type Timestamps = Type.Timestamp | Type.TimestampSecond | Type.TimestampMillisecond | Type.TimestampMicrosecond | Type.TimestampNanosecond; +/** @ignore */ +interface Timestamp_<T extends Timestamps = Timestamps> extends DataType<T> { TArray: Int32Array; TValue: number; ArrayType: TypedArrayConstructor<Int32Array> } +/** @ignore */ +class Timestamp_<T extends Timestamps = Timestamps> extends DataType<T> { + constructor(public readonly unit: TimeUnit, + public readonly timezone?: string | null) { + super(); + } + public get typeId() { return Type.Timestamp as T; } + public toString() { return `Timestamp<${TimeUnit[this.unit]}${this.timezone ? `, ${this.timezone}` : ``}>`; } + protected static [Symbol.toStringTag] = ((proto: Timestamp_) => { + (<any> proto).unit = null; + (<any> proto).timezone = null; + (<any> proto).ArrayType = Int32Array; + return proto[Symbol.toStringTag] = 'Timestamp'; + })(Timestamp_.prototype); +} + +export { Timestamp_ as Timestamp }; + +/** @ignore */ +export class TimestampSecond extends Timestamp_<Type.TimestampSecond> { constructor(timezone?: string | null) { super(TimeUnit.SECOND, timezone); } } +/** @ignore */ +export class TimestampMillisecond extends Timestamp_<Type.TimestampMillisecond> { constructor(timezone?: string | null) { super(TimeUnit.MILLISECOND, timezone); } } +/** @ignore */ +export class TimestampMicrosecond extends Timestamp_<Type.TimestampMicrosecond> { constructor(timezone?: string | null) { super(TimeUnit.MICROSECOND, timezone); } } +/** @ignore */ +export class TimestampNanosecond extends Timestamp_<Type.TimestampNanosecond> { constructor(timezone?: string | null) { super(TimeUnit.NANOSECOND, timezone); } } + +/** @ignore */ +type Intervals = Type.Interval | Type.IntervalDayTime | Type.IntervalYearMonth; +/** @ignore */ +interface Interval_<T extends Intervals = Intervals> extends DataType<T> { TArray: Int32Array; TValue: Int32Array; ArrayType: TypedArrayConstructor<Int32Array> } +/** @ignore */ +class Interval_<T extends Intervals = Intervals> extends DataType<T> { + constructor(public readonly unit: IntervalUnit) { + super(); + } + public get typeId() { return Type.Interval as T; } + public toString() { return `Interval<${IntervalUnit[this.unit]}>`; } + protected static [Symbol.toStringTag] = ((proto: Interval_) => { + (<any> proto).unit = null; + (<any> proto).ArrayType = Int32Array; + return proto[Symbol.toStringTag] = 'Interval'; + })(Interval_.prototype); +} + +export { Interval_ as Interval }; + +/** @ignore */ +export class IntervalDayTime extends Interval_<Type.IntervalDayTime> { constructor() { super(IntervalUnit.DAY_TIME); } } +/** @ignore */ +export class IntervalYearMonth extends Interval_<Type.IntervalYearMonth> { constructor() { super(IntervalUnit.YEAR_MONTH); } } + +/** @ignore */ +export interface List<T extends DataType = any> extends DataType<Type.List, { [0]: T }> { TArray: IterableArrayLike<T>; TValue: V<T> } +/** @ignore */ +export class List<T extends DataType = any> extends DataType<Type.List, { [0]: T }> { + constructor(child: Field<T>) { + super(); + this.children = [child]; + } + public readonly children: Field<T>[]; + public get typeId() { return Type.List as Type.List; } + public toString() { return `List<${this.valueType}>`; } + public get valueType(): T { return this.children[0].type as T; } + public get valueField(): Field<T> { return this.children[0] as Field<T>; } + public get ArrayType(): T['ArrayType'] { return this.valueType.ArrayType; } + protected static [Symbol.toStringTag] = ((proto: List) => { + (<any> proto).children = null; + return proto[Symbol.toStringTag] = 'List'; + })(List.prototype); +} + +/** @ignore */ +export interface Struct<T extends { [key: string]: DataType } = any> extends DataType<Type.Struct> { TArray: IterableArrayLike<RowLike<T>>; TValue: RowLike<T>; dataTypes: T } +/** @ignore */ +export class Struct<T extends { [key: string]: DataType } = any> extends DataType<Type.Struct, T> { + public readonly children: Field<T[keyof T]>[]; + constructor(children: Field<T[keyof T]>[]) { + super(); + this.children = children; + } + public get typeId() { return Type.Struct as Type.Struct; } + public toString() { return `Struct<{${this.children.map((f) => `${f.name}:${f.type}`).join(`, `)}}>`; } + protected static [Symbol.toStringTag] = ((proto: Struct) => { + (<any> proto).children = null; + return proto[Symbol.toStringTag] = 'Struct'; + })(Struct.prototype); +} + +/** @ignore */ +type Unions = Type.Union | Type.DenseUnion | Type.SparseUnion; +/** @ignore */ +interface Union_<T extends Unions = Unions> extends DataType<T> { TArray: Int8Array; TValue: any; ArrayType: TypedArrayConstructor<Int8Array> } +/** @ignore */ +class Union_<T extends Unions = Unions> extends DataType<T> { + public readonly mode: UnionMode; + public readonly typeIds: Int32Array; + public readonly children: Field<any>[]; + public readonly typeIdToChildIndex: { [key: number]: number }; + constructor(mode: UnionMode, + typeIds: number[] | Int32Array, + children: Field<any>[]) { + super(); + this.mode = mode; + this.children = children; + this.typeIds = typeIds = Int32Array.from(typeIds); + this.typeIdToChildIndex = typeIds.reduce((typeIdToChildIndex, typeId, idx) => { + return (typeIdToChildIndex[typeId] = idx) && typeIdToChildIndex || typeIdToChildIndex; + }, Object.create(null) as { [key: number]: number }); + } + public get typeId() { return Type.Union as T; } + public toString() { + return `${this[Symbol.toStringTag]}<${ + this.children.map((x) => `${x.type}`).join(` | `) + }>`; +} + protected static [Symbol.toStringTag] = ((proto: Union_) => { + (<any> proto).mode = null; + (<any> proto).typeIds = null; + (<any> proto).children = null; + (<any> proto).typeIdToChildIndex = null; + (<any> proto).ArrayType = Int8Array; + return proto[Symbol.toStringTag] = 'Union'; + })(Union_.prototype); +} + +export { Union_ as Union }; + +/** @ignore */ +export class DenseUnion extends Union_<Type.DenseUnion> { + constructor(typeIds: number[] | Int32Array, children: Field[]) { + super(UnionMode.Dense, typeIds, children); + } +} + +/** @ignore */ +export class SparseUnion extends Union_<Type.SparseUnion> { + constructor(typeIds: number[] | Int32Array, children: Field[]) { + super(UnionMode.Sparse, typeIds, children); + } +} + +/** @ignore */ +export interface FixedSizeBinary extends DataType<Type.FixedSizeBinary> { TArray: Uint8Array; TValue: Uint8Array; ArrayType: TypedArrayConstructor<Uint8Array> } +/** @ignore */ +export class FixedSizeBinary extends DataType<Type.FixedSizeBinary> { + constructor(public readonly byteWidth: number) { + super(); + } + public get typeId() { return Type.FixedSizeBinary as Type.FixedSizeBinary; } + public toString() { return `FixedSizeBinary[${this.byteWidth}]`; } + protected static [Symbol.toStringTag] = ((proto: FixedSizeBinary) => { + (<any> proto).byteWidth = null; + (<any> proto).ArrayType = Uint8Array; + return proto[Symbol.toStringTag] = 'FixedSizeBinary'; + })(FixedSizeBinary.prototype); +} + +/** @ignore */ +export interface FixedSizeList<T extends DataType = any> extends DataType<Type.FixedSizeList> { TArray: IterableArrayLike<T['TArray']>; TValue: V<T> } +/** @ignore */ +export class FixedSizeList<T extends DataType = any> extends DataType<Type.FixedSizeList, { [0]: T }> { + public readonly children: Field<T>[]; + constructor(public readonly listSize: number, child: Field<T>) { + super(); + this.children = [child]; + } + public get typeId() { return Type.FixedSizeList as Type.FixedSizeList; } + public get valueType(): T { return this.children[0].type as T; } + public get valueField(): Field<T> { return this.children[0] as Field<T>; } + public get ArrayType(): T['ArrayType'] { return this.valueType.ArrayType; } + public toString() { return `FixedSizeList[${this.listSize}]<${this.valueType}>`; } + protected static [Symbol.toStringTag] = ((proto: FixedSizeList) => { + (<any> proto).children = null; + (<any> proto).listSize = null; + return proto[Symbol.toStringTag] = 'FixedSizeList'; + })(FixedSizeList.prototype); +} + +/** @ignore */ +export interface Map_<TKey extends DataType = any, TValue extends DataType = any> extends DataType<Type.Map> { + TArray: IterableArrayLike<Map<TKey['TValue'], TValue['TValue'] | null>>; + TChild: Struct<{ key: TKey; value: TValue }>; + TValue: MapLike<TKey, TValue>; +} + +/** @ignore */ +export class Map_<TKey extends DataType = any, TValue extends DataType = any> extends DataType<Type.Map> { + constructor(child: Field<Struct<{ key: TKey; value: TValue }>>, keysSorted = false) { + super(); + this.children = [child]; + this.keysSorted = keysSorted; + } + public readonly keysSorted: boolean; + public readonly children: Field<Struct<{ key: TKey; value: TValue }>>[]; + public get typeId() { return Type.Map as Type.Map; } + public get keyType(): TKey { return this.children[0].type.children[0].type as TKey; } + public get valueType(): TValue { return this.children[0].type.children[1].type as TValue; } + public toString() { return `Map<{${this.children[0].type.children.map((f) => `${f.name}:${f.type}`).join(`, `)}}>`; } + protected static [Symbol.toStringTag] = ((proto: Map_) => { + (<any> proto).children = null; + (<any> proto).keysSorted = null; + return proto[Symbol.toStringTag] = 'Map_'; + })(Map_.prototype); +} + +/** @ignore */ +const getId = ((atomicDictionaryId) => () => ++atomicDictionaryId)(-1); + +/** @ignore */ +export type TKeys = Int8 | Int16 | Int32 | Uint8 | Uint16 | Uint32; + +/** @ignore */ +export interface Dictionary<T extends DataType = any, TKey extends TKeys = TKeys> extends DataType<Type.Dictionary> { TArray: TKey['TArray']; TValue: T['TValue'] } +/** @ignore */ +export class Dictionary<T extends DataType = any, TKey extends TKeys = TKeys> extends DataType<Type.Dictionary> { + public readonly id: number; + public readonly indices: TKey; + public readonly dictionary: T; + public readonly isOrdered: boolean; + constructor(dictionary: T, indices: TKey, id?: Long | number | null, isOrdered?: boolean | null) { + super(); + this.indices = indices; + this.dictionary = dictionary; + this.isOrdered = isOrdered || false; + this.id = id == null ? getId() : typeof id === 'number' ? id : id.low; + } + public get typeId() { return Type.Dictionary as Type.Dictionary; } + public get children() { return this.dictionary.children; } + public get valueType(): T { return this.dictionary as T; } + public get ArrayType(): T['ArrayType'] { return this.dictionary.ArrayType; } + public toString() { return `Dictionary<${this.indices}, ${this.dictionary}>`; } + protected static [Symbol.toStringTag] = ((proto: Dictionary) => { + (<any> proto).id = null; + (<any> proto).indices = null; + (<any> proto).isOrdered = null; + (<any> proto).dictionary = null; + return proto[Symbol.toStringTag] = 'Dictionary'; + })(Dictionary.prototype); +} + +/** @ignore */ +export interface IterableArrayLike<T = any> extends ArrayLike<T>, Iterable<T> {} +/** @ignore */ +export type FloatArray = Uint16Array | Float32Array | Float64Array; +/** @ignore */ +export type IntArray = Int8Array | Int16Array | Int32Array | Uint8Array | Uint16Array | Uint32Array; + +/** @ignore */ +export function strideForType(type: DataType) { + const t: any = type; + switch (type.typeId) { + case Type.Decimal: return 4; + case Type.Timestamp: return 2; + case Type.Date: return 1 + (t as Date_).unit; + case Type.Interval: return 1 + (t as Interval_).unit; + case Type.Int: return 1 + +((t as Int_).bitWidth > 32); + case Type.Time: return 1 + +((t as Time_).bitWidth > 32); + case Type.FixedSizeList: return (t as FixedSizeList).listSize; + case Type.FixedSizeBinary: return (t as FixedSizeBinary).byteWidth; + default: return 1; + } +} diff --git a/src/arrow/js/src/util/args.ts b/src/arrow/js/src/util/args.ts new file mode 100644 index 000000000..25f571999 --- /dev/null +++ b/src/arrow/js/src/util/args.ts @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import { Field } from '../schema'; +import { Column } from '../column'; +import { Vector } from '../vector'; +import { DataType, Float32, Float64, FloatArray, IntArray, Int16, Int32, Int64, Int8, Uint16, Uint32, Uint64, Uint8 } from '../type'; +import { Chunked } from '../vector/chunked'; +import { BigIntArray, TypedArray as TypedArray_ } from '../interfaces'; +import { FloatArrayCtor } from '../vector/float'; +import { IntArrayCtor } from '../vector/int'; + +type RecordBatchCtor = typeof import('../recordbatch').RecordBatch; + +const isArray = Array.isArray; + +type TypedArray = Exclude<TypedArray_ | BigIntArray, Uint8ClampedArray>; + +/** @ignore */ +export function isTypedArray(arr: any): arr is TypedArray { + return ArrayBuffer.isView(arr) && 'BYTES_PER_ELEMENT' in arr; +} + + +/** @ignore */ +type ArrayCtor = FloatArrayCtor | IntArrayCtor; + +/** @ignore */ +export function arrayTypeToDataType(ctor: ArrayCtor) { + switch (ctor) { + case Int8Array: return Int8; + case Int16Array: return Int16; + case Int32Array: return Int32; + case BigInt64Array: return Int64; + case Uint8Array: return Uint8; + case Uint16Array: return Uint16; + case Uint32Array: return Uint32; + case BigUint64Array: return Uint64; + case Float32Array: return Float32; + case Float64Array: return Float64; + default: return null; + } +} + +/** @ignore */ +function vectorFromTypedArray(array: TypedArray): Vector { + const ArrowType = arrayTypeToDataType(array.constructor as ArrayCtor); + if (!ArrowType) { + throw new TypeError('Unrecognized Array input'); + } + const type = new ArrowType(); + const data = Data.new(type, 0, array.length, 0, [undefined, array as IntArray | FloatArray]); + return Vector.new(data); +} + +/** @ignore */ +export const selectArgs = <T>(Ctor: any, vals: any[]) => _selectArgs(Ctor, vals, [], 0) as T[]; +/** @ignore */ +export const selectColumnArgs = <T extends { [key: string]: DataType }>(args: any[]) => { + const [fields, values] = _selectFieldArgs<T>(args, [[], []]); + return values.map((x, i) => + x instanceof Column ? Column.new(x.field.clone(fields[i]), x) : + x instanceof Vector ? Column.new(fields[i], x) as Column<T[keyof T]> : + isTypedArray(x) ? Column.new(fields[i], vectorFromTypedArray(x)) as Column<T[keyof T]> : + Column.new(fields[i], [] as Vector<T[keyof T]>[])); +}; + +/** @ignore */ +export const selectFieldArgs = <T extends { [key: string]: DataType }>(args: any[]) => _selectFieldArgs<T>(args, [[], []]); +/** @ignore */ +export const selectChunkArgs = <T>(Ctor: any, vals: any[]) => _selectChunkArgs(Ctor, vals, [], 0) as T[]; +/** @ignore */ +export const selectVectorChildrenArgs = <T extends Vector>(Ctor: RecordBatchCtor, vals: any[]) => _selectVectorChildrenArgs(Ctor, vals, [], 0) as T[]; +/** @ignore */ +export const selectColumnChildrenArgs = <T extends Column>(Ctor: RecordBatchCtor, vals: any[]) => _selectColumnChildrenArgs(Ctor, vals, [], 0) as T[]; + +/** @ignore */ +function _selectArgs<T>(Ctor: any, vals: any[], res: T[], idx: number) { + let value: any, j = idx; + let i = -1; + const n = vals.length; + while (++i < n) { + if (isArray(value = vals[i])) { + j = _selectArgs(Ctor, value, res, j).length; + } else if (value instanceof Ctor) { res[j++] = value; } + } + return res; +} + +/** @ignore */ +function _selectChunkArgs<T>(Ctor: any, vals: any[], res: T[], idx: number) { + let value: any, j = idx; + let i = -1; + const n = vals.length; + while (++i < n) { + if (isArray(value = vals[i])) { + j = _selectChunkArgs(Ctor, value, res, j).length; + } else if (value instanceof Chunked) { + j = _selectChunkArgs(Ctor, value.chunks, res, j).length; + } else if (value instanceof Ctor) { res[j++] = value; } + } + return res; +} + +/** @ignore */ +function _selectVectorChildrenArgs<T extends Vector>(Ctor: RecordBatchCtor, vals: any[], res: T[], idx: number) { + let value: any, j = idx; + let i = -1; + const n = vals.length; + while (++i < n) { + if (isArray(value = vals[i])) { + j = _selectVectorChildrenArgs(Ctor, value, res, j).length; + } else if (value instanceof Ctor) { + j = _selectArgs(Vector, value.schema.fields.map((_, i) => value.getChildAt(i)!), res, j).length; + } else if (value instanceof Vector) { res[j++] = value as T; } + } + return res; +} + +/** @ignore */ +function _selectColumnChildrenArgs<T extends Column>(Ctor: RecordBatchCtor, vals: any[], res: T[], idx: number) { + let value: any, j = idx; + let i = -1; + const n = vals.length; + while (++i < n) { + if (isArray(value = vals[i])) { + j = _selectColumnChildrenArgs(Ctor, value, res, j).length; + } else if (value instanceof Ctor) { + j = _selectArgs(Column, value.schema.fields.map((f, i) => Column.new(f, value.getChildAt(i)!)), res, j).length; + } else if (value instanceof Column) { res[j++] = value as T; } + } + return res; +} + +/** @ignore */ +const toKeysAndValues = (xs: [any[], any[]], [k, v]: [any, any], i: number) => (xs[0][i] = k, xs[1][i] = v, xs); + +/** @ignore */ +function _selectFieldArgs<T extends { [key: string]: DataType }>(vals: any[], ret: [Field<T[keyof T]>[], (Vector<T[keyof T]> | TypedArray)[]]): [Field<T[keyof T]>[], (T[keyof T] | Vector<T[keyof T]> | TypedArray)[]] { + let keys: any[]; + let n: number; + switch (n = vals.length) { + case 0: return ret; + case 1: + keys = ret[0]; + if (!(vals[0])) { return ret; } + if (isArray(vals[0])) { return _selectFieldArgs(vals[0], ret); } + if (!(vals[0] instanceof Data || vals[0] instanceof Vector || isTypedArray(vals[0]) || vals[0] instanceof DataType)) { + [keys, vals] = Object.entries(vals[0]).reduce(toKeysAndValues, ret); + } + break; + default: + !isArray(keys = vals[n - 1]) + ? (vals = isArray(vals[0]) ? vals[0] : vals, keys = []) + : (vals = isArray(vals[0]) ? vals[0] : vals.slice(0, n - 1)); + } + + let fieldIndex = -1; + let valueIndex = -1; + let idx = -1; + const len = vals.length; + let field: number | string | Field<T[keyof T]>; + let val: Vector<T[keyof T]> | Data<T[keyof T]>; + const [fields, values] = ret as [Field<T[keyof T]>[], any[]]; + + while (++idx < len) { + val = vals[idx]; + if (val instanceof Column && (values[++valueIndex] = val)) { + fields[++fieldIndex] = val.field.clone(keys[idx], val.type, true); + } else { + ({ [idx]: field = idx } = keys); + if (val instanceof DataType && (values[++valueIndex] = val)) { + fields[++fieldIndex] = Field.new(field, val as DataType, true) as Field<T[keyof T]>; + } else if (val?.type && (values[++valueIndex] = val)) { + val instanceof Data && (values[valueIndex] = val = Vector.new(val) as Vector); + fields[++fieldIndex] = Field.new(field, val.type, true) as Field<T[keyof T]>; + } + } + } + return ret; +} diff --git a/src/arrow/js/src/util/bit.ts b/src/arrow/js/src/util/bit.ts new file mode 100644 index 000000000..e4c3d267e --- /dev/null +++ b/src/arrow/js/src/util/bit.ts @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/** @ignore */ +export function getBool(_data: any, _index: number, byte: number, bit: number) { + return (byte & 1 << bit) !== 0; +} + +/** @ignore */ +export function getBit(_data: any, _index: number, byte: number, bit: number): 0 | 1 { + return (byte & 1 << bit) >> bit as (0 | 1); +} + +/** @ignore */ +export function setBool(bytes: Uint8Array, index: number, value: any) { + return value ? + !!(bytes[index >> 3] |= (1 << (index % 8))) || true : + !(bytes[index >> 3] &= ~(1 << (index % 8))) && false ; +} + +/** @ignore */ +export function truncateBitmap(offset: number, length: number, bitmap: Uint8Array) { + const alignedSize = (bitmap.byteLength + 7) & ~7; + if (offset > 0 || bitmap.byteLength < alignedSize) { + const bytes = new Uint8Array(alignedSize); + // If the offset is a multiple of 8 bits, it's safe to slice the bitmap + bytes.set(offset % 8 === 0 ? bitmap.subarray(offset >> 3) : + // Otherwise iterate each bit from the offset and return a new one + packBools(new BitIterator(bitmap, offset, length, null, getBool)).subarray(0, alignedSize)); + return bytes; + } + return bitmap; +} + +/** @ignore */ +export function packBools(values: Iterable<any>) { + const xs: number[] = []; + let i = 0, bit = 0, byte = 0; + for (const value of values) { + value && (byte |= 1 << bit); + if (++bit === 8) { + xs[i++] = byte; + byte = bit = 0; + } + } + if (i === 0 || bit > 0) { xs[i++] = byte; } + const b = new Uint8Array((xs.length + 7) & ~7); + b.set(xs); + return b; +} + +/** @ignore */ +export class BitIterator<T> implements IterableIterator<T> { + bit: number; + byte: number; + byteIndex: number; + index: number; + + constructor( + private bytes: Uint8Array, + begin: number, + private length: number, + private context: any, + private get: (context: any, index: number, byte: number, bit: number) => T + ) { + this.bit = begin % 8; + this.byteIndex = begin >> 3; + this.byte = bytes[this.byteIndex++]; + this.index = 0; + } + + next(): IteratorResult<T> { + if (this.index < this.length) { + if (this.bit === 8) { + this.bit = 0; + this.byte = this.bytes[this.byteIndex++]; + } + return { + value: this.get(this.context, this.index++, this.byte, this.bit++) + }; + } + return { done: true, value: null }; + } + + [Symbol.iterator]() { + return this; + } +} + +/** + * Compute the population count (the number of bits set to 1) for a range of bits in a Uint8Array. + * @param vector The Uint8Array of bits for which to compute the population count. + * @param lhs The range's left-hand side (or start) bit + * @param rhs The range's right-hand side (or end) bit + */ +/** @ignore */ +export function popcnt_bit_range(data: Uint8Array, lhs: number, rhs: number): number { + if (rhs - lhs <= 0) { return 0; } + // If the bit range is less than one byte, sum the 1 bits in the bit range + if (rhs - lhs < 8) { + let sum = 0; + for (const bit of new BitIterator(data, lhs, rhs - lhs, data, getBit)) { + sum += bit; + } + return sum; + } + // Get the next lowest multiple of 8 from the right hand side + const rhsInside = rhs >> 3 << 3; + // Get the next highest multiple of 8 from the left hand side + const lhsInside = lhs + (lhs % 8 === 0 ? 0 : 8 - lhs % 8); + return ( + // Get the popcnt of bits between the left hand side, and the next highest multiple of 8 + popcnt_bit_range(data, lhs, lhsInside) + + // Get the popcnt of bits between the right hand side, and the next lowest multiple of 8 + popcnt_bit_range(data, rhsInside, rhs) + + // Get the popcnt of all bits between the left and right hand sides' multiples of 8 + popcnt_array(data, lhsInside >> 3, (rhsInside - lhsInside) >> 3) + ); +} + +/** @ignore */ +export function popcnt_array(arr: ArrayBufferView, byteOffset?: number, byteLength?: number) { + let cnt = 0, pos = byteOffset! | 0; + const view = new DataView(arr.buffer, arr.byteOffset, arr.byteLength); + const len = byteLength === void 0 ? arr.byteLength : pos + byteLength; + while (len - pos >= 4) { + cnt += popcnt_uint32(view.getUint32(pos)); + pos += 4; + } + while (len - pos >= 2) { + cnt += popcnt_uint32(view.getUint16(pos)); + pos += 2; + } + while (len - pos >= 1) { + cnt += popcnt_uint32(view.getUint8(pos)); + pos += 1; + } + return cnt; +} + +/** @ignore */ +export function popcnt_uint32(uint32: number): number { + let i = uint32 | 0; + i = i - ((i >>> 1) & 0x55555555); + i = (i & 0x33333333) + ((i >>> 2) & 0x33333333); + return (((i + (i >>> 4)) & 0x0F0F0F0F) * 0x01010101) >>> 24; +} diff --git a/src/arrow/js/src/util/bn.ts b/src/arrow/js/src/util/bn.ts new file mode 100644 index 000000000..7c71969a4 --- /dev/null +++ b/src/arrow/js/src/util/bn.ts @@ -0,0 +1,231 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { ArrayBufferViewInput, toArrayBufferView } from './buffer'; +import { TypedArray, TypedArrayConstructor } from '../interfaces'; +import { BigIntArray, BigIntArrayConstructor } from '../interfaces'; +import { BigIntAvailable, BigInt64Array, BigUint64Array } from './compat'; + +/** @ignore */ +export const isArrowBigNumSymbol = Symbol.for('isArrowBigNum'); + +/** @ignore */ type BigNumArray = IntArray | UintArray; +/** @ignore */ type IntArray = Int8Array | Int16Array | Int32Array; +/** @ignore */ type UintArray = Uint8Array | Uint16Array | Uint32Array | Uint8ClampedArray; + +/** @ignore */ +function BigNum(this: any, x: any, ...xs: any) { + if (xs.length === 0) { + return Object.setPrototypeOf(toArrayBufferView(this['TypedArray'], x), this.constructor.prototype); + } + return Object.setPrototypeOf(new this['TypedArray'](x, ...xs), this.constructor.prototype); +} + +BigNum.prototype[isArrowBigNumSymbol] = true; +BigNum.prototype.toJSON = function<T extends BN<BigNumArray>>(this: T) { return `"${bignumToString(this)}"`; }; +BigNum.prototype.valueOf = function<T extends BN<BigNumArray>>(this: T) { return bignumToNumber(this); }; +BigNum.prototype.toString = function<T extends BN<BigNumArray>>(this: T) { return bignumToString(this); }; +BigNum.prototype[Symbol.toPrimitive] = function<T extends BN<BigNumArray>>(this: T, hint: 'string' | 'number' | 'default' = 'default') { + switch (hint) { + case 'number': return bignumToNumber(this); + case 'string': return bignumToString(this); + case 'default': return bignumToBigInt(this); + } + // @ts-ignore + return bignumToString(this); +}; + +/** @ignore */ +type TypedArrayConstructorArgs = + [number | void] | + [Iterable<number> | Iterable<bigint>] | + [ArrayBufferLike, number | void, number | void] ; + +/** @ignore */ +function SignedBigNum(this: any, ...args: TypedArrayConstructorArgs) { return BigNum.apply(this, args); } +/** @ignore */ +function UnsignedBigNum(this: any, ...args: TypedArrayConstructorArgs) { return BigNum.apply(this, args); } +/** @ignore */ +function DecimalBigNum(this: any, ...args: TypedArrayConstructorArgs) { return BigNum.apply(this, args); } + +Object.setPrototypeOf(SignedBigNum.prototype, Object.create(Int32Array.prototype)); +Object.setPrototypeOf(UnsignedBigNum.prototype, Object.create(Uint32Array.prototype)); +Object.setPrototypeOf(DecimalBigNum.prototype, Object.create(Uint32Array.prototype)); +Object.assign(SignedBigNum.prototype, BigNum.prototype, { 'constructor': SignedBigNum, 'signed': true, 'TypedArray': Int32Array, 'BigIntArray': BigInt64Array }); +Object.assign(UnsignedBigNum.prototype, BigNum.prototype, { 'constructor': UnsignedBigNum, 'signed': false, 'TypedArray': Uint32Array, 'BigIntArray': BigUint64Array }); +Object.assign(DecimalBigNum.prototype, BigNum.prototype, { 'constructor': DecimalBigNum, 'signed': true, 'TypedArray': Uint32Array, 'BigIntArray': BigUint64Array }); + +/** @ignore */ +function bignumToNumber<T extends BN<BigNumArray>>(bn: T) { + const { buffer, byteOffset, length, 'signed': signed } = bn; + const words = new Int32Array(buffer, byteOffset, length); + let number = 0, i = 0; + const n = words.length; + let hi, lo; + while (i < n) { + lo = words[i++]; + hi = words[i++]; + signed || (hi = hi >>> 0); + number += (lo >>> 0) + (hi * (i ** 32)); + } + return number; +} + +/** @ignore */ +export let bignumToString: { <T extends BN<BigNumArray>>(a: T): string }; +/** @ignore */ +export let bignumToBigInt: { <T extends BN<BigNumArray>>(a: T): bigint }; + +if (!BigIntAvailable) { + bignumToString = decimalToString; + bignumToBigInt = <any> bignumToString; +} else { + bignumToBigInt = (<T extends BN<BigNumArray>>(a: T) => a.byteLength === 8 ? new a['BigIntArray'](a.buffer, a.byteOffset, 1)[0] : <any>decimalToString(a)); + bignumToString = (<T extends BN<BigNumArray>>(a: T) => a.byteLength === 8 ? `${new a['BigIntArray'](a.buffer, a.byteOffset, 1)[0]}` : decimalToString(a)); +} + +/** @ignore */ +function decimalToString<T extends BN<BigNumArray>>(a: T) { + let digits = ''; + const base64 = new Uint32Array(2); + let base32 = new Uint16Array(a.buffer, a.byteOffset, a.byteLength / 2); + const checks = new Uint32Array((base32 = new Uint16Array(base32).reverse()).buffer); + let i = -1; + const n = base32.length - 1; + do { + for (base64[0] = base32[i = 0]; i < n;) { + base32[i++] = base64[1] = base64[0] / 10; + base64[0] = ((base64[0] - base64[1] * 10) << 16) + base32[i]; + } + base32[i] = base64[1] = base64[0] / 10; + base64[0] = base64[0] - base64[1] * 10; + digits = `${base64[0]}${digits}`; + } while (checks[0] || checks[1] || checks[2] || checks[3]); + return digits ? digits : `0`; +} + +/** @ignore */ +export class BN<T extends BigNumArray> { + /** @nocollapse */ + public static new<T extends BigNumArray>(num: T, isSigned?: boolean): (T & BN<T>) { + switch (isSigned) { + case true: return new (<any> SignedBigNum)(num) as (T & BN<T>); + case false: return new (<any> UnsignedBigNum)(num) as (T & BN<T>); + } + switch (num.constructor) { + case Int8Array: + case Int16Array: + case Int32Array: + case BigInt64Array: + return new (<any> SignedBigNum)(num) as (T & BN<T>); + } + if (num.byteLength === 16) { + return new (<any> DecimalBigNum)(num) as (T & BN<T>); + } + return new (<any> UnsignedBigNum)(num) as (T & BN<T>); + } + /** @nocollapse */ + public static signed<T extends IntArray>(num: T): (T & BN<T>) { + return new (<any> SignedBigNum)(num) as (T & BN<T>); + } + /** @nocollapse */ + public static unsigned<T extends UintArray>(num: T): (T & BN<T>) { + return new (<any> UnsignedBigNum)(num) as (T & BN<T>); + } + /** @nocollapse */ + public static decimal<T extends UintArray>(num: T): (T & BN<T>) { + return new (<any> DecimalBigNum)(num) as (T & BN<T>); + } + constructor(num: T, isSigned?: boolean) { + return BN.new(num, isSigned) as any; + } +} + +/** @ignore */ +export interface BN<T extends BigNumArray> extends TypedArrayLike<T> { + + new<T extends ArrayBufferViewInput>(buffer: T, signed?: boolean): T; + + readonly signed: boolean; + readonly TypedArray: TypedArrayConstructor<TypedArray>; + readonly BigIntArray: BigIntArrayConstructor<BigIntArray>; + + [Symbol.toStringTag]: + 'Int8Array' | + 'Int16Array' | + 'Int32Array' | + 'Uint8Array' | + 'Uint16Array' | + 'Uint32Array' | + 'Uint8ClampedArray'; + + /** + * Convert the bytes to their (positive) decimal representation for printing + */ + toString(): string; + /** + * Down-convert the bytes to a 53-bit precision integer. Invoked by JS for + * arithmetic operators, like `+`. Easy (and unsafe) way to convert BN to + * number via `+bn_inst` + */ + valueOf(): number; + /** + * Return the JSON representation of the bytes. Must be wrapped in double-quotes, + * so it's compatible with JSON.stringify(). + */ + toJSON(): string; + [Symbol.toPrimitive](hint?: any): number | string | bigint; +} + +/** @ignore */ +interface TypedArrayLike<T extends BigNumArray> { + + readonly length: number; + readonly buffer: ArrayBuffer; + readonly byteLength: number; + readonly byteOffset: number; + readonly BYTES_PER_ELEMENT: number; + + includes(searchElement: number, fromIndex?: number | undefined): boolean; + copyWithin(target: number, start: number, end?: number | undefined): this; + every(callbackfn: (value: number, index: number, array: T) => boolean, thisArg?: any): boolean; + fill(value: number, start?: number | undefined, end?: number | undefined): this; + filter(callbackfn: (value: number, index: number, array: T) => boolean, thisArg?: any): T; + find(predicate: (value: number, index: number, obj: T) => boolean, thisArg?: any): number | undefined; + findIndex(predicate: (value: number, index: number, obj: T) => boolean, thisArg?: any): number; + forEach(callbackfn: (value: number, index: number, array: T) => void, thisArg?: any): void; + indexOf(searchElement: number, fromIndex?: number | undefined): number; + join(separator?: string | undefined): string; + lastIndexOf(searchElement: number, fromIndex?: number | undefined): number; + map(callbackfn: (value: number, index: number, array: T) => number, thisArg?: any): T; + reduce(callbackfn: (previousValue: number, currentValue: number, currentIndex: number, array: T) => number): number; + reduce(callbackfn: (previousValue: number, currentValue: number, currentIndex: number, array: T) => number, initialValue: number): number; + reduce<U>(callbackfn: (previousValue: U, currentValue: number, currentIndex: number, array: T) => U, initialValue: U): U; + reduceRight(callbackfn: (previousValue: number, currentValue: number, currentIndex: number, array: T) => number): number; + reduceRight(callbackfn: (previousValue: number, currentValue: number, currentIndex: number, array: T) => number, initialValue: number): number; + reduceRight<U>(callbackfn: (previousValue: U, currentValue: number, currentIndex: number, array: T) => U, initialValue: U): U; + reverse(): T; + set(array: ArrayLike<number>, offset?: number | undefined): void; + slice(start?: number | undefined, end?: number | undefined): T; + some(callbackfn: (value: number, index: number, array: T) => boolean, thisArg?: any): boolean; + sort(compareFn?: ((a: number, b: number) => number) | undefined): this; + subarray(begin: number, end?: number | undefined): T; + toLocaleString(): string; + entries(): IterableIterator<[number, number]>; + keys(): IterableIterator<number>; + values(): IterableIterator<number>; +} diff --git a/src/arrow/js/src/util/buffer.ts b/src/arrow/js/src/util/buffer.ts new file mode 100644 index 000000000..86dae86c6 --- /dev/null +++ b/src/arrow/js/src/util/buffer.ts @@ -0,0 +1,235 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { flatbuffers } from 'flatbuffers'; +import { encodeUtf8 } from '../util/utf8'; +import ByteBuffer = flatbuffers.ByteBuffer; +import { TypedArray, TypedArrayConstructor } from '../interfaces'; +import { BigIntArray, BigIntArrayConstructor } from '../interfaces'; +import { isPromise, isIterable, isAsyncIterable, isIteratorResult, BigInt64Array, BigUint64Array } from './compat'; + +/** @ignore */ +const SharedArrayBuf = (typeof SharedArrayBuffer !== 'undefined' ? SharedArrayBuffer : ArrayBuffer); + +/** @ignore */ +function collapseContiguousByteRanges(chunks: Uint8Array[]) { + const result = chunks[0] ? [chunks[0]] : []; + let xOffset: number, yOffset: number, xLen: number, yLen: number; + for (let x, y, i = 0, j = 0, n = chunks.length; ++i < n;) { + x = result[j]; + y = chunks[i]; + // continue if x and y don't share the same underlying ArrayBuffer, or if x isn't before y + if (!x || !y || x.buffer !== y.buffer || y.byteOffset < x.byteOffset) { + y && (result[++j] = y); + continue; + } + ({ byteOffset: xOffset, byteLength: xLen } = x); + ({ byteOffset: yOffset, byteLength: yLen } = y); + // continue if the byte ranges of x and y aren't contiguous + if ((xOffset + xLen) < yOffset || (yOffset + yLen) < xOffset) { + y && (result[++j] = y); + continue; + } + result[j] = new Uint8Array(x.buffer, xOffset, yOffset - xOffset + yLen); + } + return result; +} + +/** @ignore */ +export function memcpy<TTarget extends ArrayBufferView, TSource extends ArrayBufferView>(target: TTarget, source: TSource, targetByteOffset = 0, sourceByteLength = source.byteLength) { + const targetByteLength = target.byteLength; + const dst = new Uint8Array(target.buffer, target.byteOffset, targetByteLength); + const src = new Uint8Array(source.buffer, source.byteOffset, Math.min(sourceByteLength, targetByteLength)); + dst.set(src, targetByteOffset); + return target; +} + +/** @ignore */ +export function joinUint8Arrays(chunks: Uint8Array[], size?: number | null): [Uint8Array, Uint8Array[], number] { + // collapse chunks that share the same underlying ArrayBuffer and whose byte ranges overlap, + // to avoid unnecessarily copying the bytes to do this buffer join. This is a common case during + // streaming, where we may be reading partial byte ranges out of the same underlying ArrayBuffer + const result = collapseContiguousByteRanges(chunks); + const byteLength = result.reduce((x, b) => x + b.byteLength, 0); + let source: Uint8Array, sliced: Uint8Array, buffer: Uint8Array | void; + let offset = 0, index = -1; + const length = Math.min(size || Infinity, byteLength); + for (let n = result.length; ++index < n;) { + source = result[index]; + sliced = source.subarray(0, Math.min(source.length, length - offset)); + if (length <= (offset + sliced.length)) { + if (sliced.length < source.length) { + result[index] = source.subarray(sliced.length); + } else if (sliced.length === source.length) { index++; } + buffer ? memcpy(buffer, sliced, offset) : (buffer = sliced); + break; + } + memcpy(buffer || (buffer = new Uint8Array(length)), sliced, offset); + offset += sliced.length; + } + return [buffer || new Uint8Array(0), result.slice(index), byteLength - (buffer ? buffer.byteLength : 0)]; +} + +/** @ignore */ +export type ArrayBufferViewInput = ArrayBufferView | ArrayBufferLike | ArrayBufferView | Iterable<number> | ArrayLike<number> | ByteBuffer | string | null | undefined | + IteratorResult<ArrayBufferView | ArrayBufferLike | ArrayBufferView | Iterable<number> | ArrayLike<number> | ByteBuffer | string | null | undefined> | + ReadableStreamReadResult<ArrayBufferView | ArrayBufferLike | ArrayBufferView | Iterable<number> | ArrayLike<number> | ByteBuffer | string | null | undefined> ; + +/** @ignore */ +export function toArrayBufferView<T extends TypedArray>(ArrayBufferViewCtor: TypedArrayConstructor<T>, input: ArrayBufferViewInput): T; +export function toArrayBufferView<T extends BigIntArray>(ArrayBufferViewCtor: BigIntArrayConstructor<T>, input: ArrayBufferViewInput): T; +export function toArrayBufferView(ArrayBufferViewCtor: any, input: ArrayBufferViewInput) { + + let value: any = isIteratorResult(input) ? input.value : input; + + if (value instanceof ArrayBufferViewCtor) { + if (ArrayBufferViewCtor === Uint8Array) { + // Node's `Buffer` class passes the `instanceof Uint8Array` check, but we need + // a real Uint8Array, since Buffer#slice isn't the same as Uint8Array#slice :/ + return new ArrayBufferViewCtor(value.buffer, value.byteOffset, value.byteLength); + } + return value; + } + if (!value) { return new ArrayBufferViewCtor(0); } + if (typeof value === 'string') { value = encodeUtf8(value); } + if (value instanceof ArrayBuffer) { return new ArrayBufferViewCtor(value); } + if (value instanceof SharedArrayBuf) { return new ArrayBufferViewCtor(value); } + if (value instanceof ByteBuffer) { return toArrayBufferView(ArrayBufferViewCtor, value.bytes()); } + return !ArrayBuffer.isView(value) ? ArrayBufferViewCtor.from(value) : value.byteLength <= 0 ? new ArrayBufferViewCtor(0) + : new ArrayBufferViewCtor(value.buffer, value.byteOffset, value.byteLength / ArrayBufferViewCtor.BYTES_PER_ELEMENT); +} + +/** @ignore */ export const toInt8Array = (input: ArrayBufferViewInput) => toArrayBufferView(Int8Array, input); +/** @ignore */ export const toInt16Array = (input: ArrayBufferViewInput) => toArrayBufferView(Int16Array, input); +/** @ignore */ export const toInt32Array = (input: ArrayBufferViewInput) => toArrayBufferView(Int32Array, input); +/** @ignore */ export const toBigInt64Array = (input: ArrayBufferViewInput) => toArrayBufferView(BigInt64Array, input); +/** @ignore */ export const toUint8Array = (input: ArrayBufferViewInput) => toArrayBufferView(Uint8Array, input); +/** @ignore */ export const toUint16Array = (input: ArrayBufferViewInput) => toArrayBufferView(Uint16Array, input); +/** @ignore */ export const toUint32Array = (input: ArrayBufferViewInput) => toArrayBufferView(Uint32Array, input); +/** @ignore */ export const toBigUint64Array = (input: ArrayBufferViewInput) => toArrayBufferView(BigUint64Array, input); +/** @ignore */ export const toFloat32Array = (input: ArrayBufferViewInput) => toArrayBufferView(Float32Array, input); +/** @ignore */ export const toFloat64Array = (input: ArrayBufferViewInput) => toArrayBufferView(Float64Array, input); +/** @ignore */ export const toUint8ClampedArray = (input: ArrayBufferViewInput) => toArrayBufferView(Uint8ClampedArray, input); + +/** @ignore */ +type ArrayBufferViewIteratorInput = Iterable<ArrayBufferViewInput> | ArrayBufferViewInput; + +/** @ignore */ +const pump = <T extends Iterator<any> | AsyncIterator<any>>(iterator: T) => { iterator.next(); return iterator; }; + +/** @ignore */ +export function* toArrayBufferViewIterator<T extends TypedArray>(ArrayCtor: TypedArrayConstructor<T>, source: ArrayBufferViewIteratorInput) { + + const wrap = function*<T>(x: T) { yield x; }; + const buffers: Iterable<ArrayBufferViewInput> = + (typeof source === 'string') ? wrap(source) + : (ArrayBuffer.isView(source)) ? wrap(source) + : (source instanceof ArrayBuffer) ? wrap(source) + : (source instanceof SharedArrayBuf) ? wrap(source) + : !isIterable<ArrayBufferViewInput>(source) ? wrap(source) : source; + + yield* pump((function* (it: Iterator<ArrayBufferViewInput, any, number | undefined>): Generator<T, void, number | undefined> { + let r: IteratorResult<any> = <any> null; + do { + r = it.next(yield toArrayBufferView(ArrayCtor, r)); + } while (!r.done); + })(buffers[Symbol.iterator]())); + return new ArrayCtor(); +} + +/** @ignore */ export const toInt8ArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Int8Array, input); +/** @ignore */ export const toInt16ArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Int16Array, input); +/** @ignore */ export const toInt32ArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Int32Array, input); +/** @ignore */ export const toUint8ArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Uint8Array, input); +/** @ignore */ export const toUint16ArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Uint16Array, input); +/** @ignore */ export const toUint32ArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Uint32Array, input); +/** @ignore */ export const toFloat32ArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Float32Array, input); +/** @ignore */ export const toFloat64ArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Float64Array, input); +/** @ignore */ export const toUint8ClampedArrayIterator = (input: ArrayBufferViewIteratorInput) => toArrayBufferViewIterator(Uint8ClampedArray, input); + +/** @ignore */ +type ArrayBufferViewAsyncIteratorInput = AsyncIterable<ArrayBufferViewInput> | Iterable<ArrayBufferViewInput> | PromiseLike<ArrayBufferViewInput> | ArrayBufferViewInput; + +/** @ignore */ +export async function* toArrayBufferViewAsyncIterator<T extends TypedArray>(ArrayCtor: TypedArrayConstructor<T>, source: ArrayBufferViewAsyncIteratorInput): AsyncGenerator<T, T, number | undefined> { + + // if a Promise, unwrap the Promise and iterate the resolved value + if (isPromise<ArrayBufferViewInput>(source)) { + return yield* toArrayBufferViewAsyncIterator(ArrayCtor, await source); + } + + const wrap = async function*<T>(x: T) { yield await x; }; + const emit = async function* <T extends Iterable<any>>(source: T) { + yield* pump((function*(it: Iterator<any>) { + let r: IteratorResult<any> = <any> null; + do { + r = it.next(yield r?.value); + } while (!r.done); + })(source[Symbol.iterator]())); + }; + + const buffers: AsyncIterable<ArrayBufferViewInput> = + (typeof source === 'string') ? wrap(source) // if string, wrap in an AsyncIterableIterator + : (ArrayBuffer.isView(source)) ? wrap(source) // if TypedArray, wrap in an AsyncIterableIterator + : (source instanceof ArrayBuffer) ? wrap(source) // if ArrayBuffer, wrap in an AsyncIterableIterator + : (source instanceof SharedArrayBuf) ? wrap(source) // if SharedArrayBuffer, wrap in an AsyncIterableIterator + : isIterable<ArrayBufferViewInput>(source) ? emit(source) // If Iterable, wrap in an AsyncIterableIterator and compose the `next` values + : !isAsyncIterable<ArrayBufferViewInput>(source) ? wrap(source) // If not an AsyncIterable, treat as a sentinel and wrap in an AsyncIterableIterator + : source; // otherwise if AsyncIterable, use it + + yield* pump((async function* (it: AsyncIterator<ArrayBufferViewInput, any, number | undefined>): AsyncGenerator<T, void, number | undefined> { + let r: IteratorResult<any> = <any> null; + do { + r = await it.next(yield toArrayBufferView(ArrayCtor, r)); + } while (!r.done); + })(buffers[Symbol.asyncIterator]())); + return new ArrayCtor(); +} + +/** @ignore */ export const toInt8ArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Int8Array, input); +/** @ignore */ export const toInt16ArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Int16Array, input); +/** @ignore */ export const toInt32ArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Int32Array, input); +/** @ignore */ export const toUint8ArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Uint8Array, input); +/** @ignore */ export const toUint16ArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Uint16Array, input); +/** @ignore */ export const toUint32ArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Uint32Array, input); +/** @ignore */ export const toFloat32ArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Float32Array, input); +/** @ignore */ export const toFloat64ArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Float64Array, input); +/** @ignore */ export const toUint8ClampedArrayAsyncIterator = (input: ArrayBufferViewAsyncIteratorInput) => toArrayBufferViewAsyncIterator(Uint8ClampedArray, input); + +/** @ignore */ +export function rebaseValueOffsets(offset: number, length: number, valueOffsets: Int32Array) { + // If we have a non-zero offset, create a new offsets array with the values + // shifted by the start offset, such that the new start offset is 0 + if (offset !== 0) { + valueOffsets = valueOffsets.slice(0, length + 1); + for (let i = -1; ++i <= length;) { + valueOffsets[i] += offset; + } + } + return valueOffsets; +} + +/** @ignore */ +export function compareArrayLike<T extends ArrayLike<any>>(a: T, b: T) { + let i = 0; + const n = a.length; + if (n !== b.length) { return false; } + if (n > 0) { + do { if (a[i] !== b[i]) { return false; } } while (++i < n); + } + return true; +} diff --git a/src/arrow/js/src/util/compat.ts b/src/arrow/js/src/util/compat.ts new file mode 100644 index 000000000..62fcb772e --- /dev/null +++ b/src/arrow/js/src/util/compat.ts @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { ReadableInterop, ArrowJSONLike } from '../io/interfaces'; + +/** @ignore */ +type FSReadStream = import('fs').ReadStream; +/** @ignore */ +type FileHandle = import('fs').promises.FileHandle; + +/** @ignore */ +export interface Subscription { + unsubscribe: () => void; +} + +/** @ignore */ +export interface Observer<T> { + closed?: boolean; + next: (value: T) => void; + error: (err: any) => void; + complete: () => void; +} + +/** @ignore */ +export interface Observable<T> { + subscribe: (observer: Observer<T>) => Subscription; +} + +/** @ignore */ +const [BigIntCtor, BigIntAvailable] = (() => { + const BigIntUnavailableError = () => { throw new Error('BigInt is not available in this environment'); }; + function BigIntUnavailable() { throw BigIntUnavailableError(); } + BigIntUnavailable.asIntN = () => { throw BigIntUnavailableError(); }; + BigIntUnavailable.asUintN = () => { throw BigIntUnavailableError(); }; + return typeof BigInt !== 'undefined' ? [BigInt, true] : [<any> BigIntUnavailable, false]; +})() as [BigIntConstructor, boolean]; + +/** @ignore */ +const [BigInt64ArrayCtor, BigInt64ArrayAvailable] = (() => { + const BigInt64ArrayUnavailableError = () => { throw new Error('BigInt64Array is not available in this environment'); }; + class BigInt64ArrayUnavailable { + static get BYTES_PER_ELEMENT() { return 8; } + static of() { throw BigInt64ArrayUnavailableError(); } + static from() { throw BigInt64ArrayUnavailableError(); } + constructor() { throw BigInt64ArrayUnavailableError(); } + } + return typeof BigInt64Array !== 'undefined' ? [BigInt64Array, true] : [<any> BigInt64ArrayUnavailable, false]; +})() as [BigInt64ArrayConstructor, boolean]; + +/** @ignore */ +const [BigUint64ArrayCtor, BigUint64ArrayAvailable] = (() => { + const BigUint64ArrayUnavailableError = () => { throw new Error('BigUint64Array is not available in this environment'); }; + class BigUint64ArrayUnavailable { + static get BYTES_PER_ELEMENT() { return 8; } + static of() { throw BigUint64ArrayUnavailableError(); } + static from() { throw BigUint64ArrayUnavailableError(); } + constructor() { throw BigUint64ArrayUnavailableError(); } + } + return typeof BigUint64Array !== 'undefined' ? [BigUint64Array, true] : [<any> BigUint64ArrayUnavailable, false]; +})() as [BigUint64ArrayConstructor, boolean]; + +export { BigIntCtor as BigInt, BigIntAvailable }; +export { BigInt64ArrayCtor as BigInt64Array, BigInt64ArrayAvailable }; +export { BigUint64ArrayCtor as BigUint64Array, BigUint64ArrayAvailable }; + +/** @ignore */ const isNumber = (x: any) => typeof x === 'number'; +/** @ignore */ const isBoolean = (x: any) => typeof x === 'boolean'; +/** @ignore */ const isFunction = (x: any) => typeof x === 'function'; +/** @ignore */ +// eslint-disable-next-line @typescript-eslint/ban-types +export const isObject = (x: any): x is Object => x != null && Object(x) === x; + +/** @ignore */ +export const isPromise = <T = any>(x: any): x is PromiseLike<T> => { + return isObject(x) && isFunction(x.then); +}; + +/** @ignore */ +export const isObservable = <T = any>(x: any): x is Observable<T> => { + return isObject(x) && isFunction(x.subscribe); +}; + +/** @ignore */ +export const isIterable = <T = any>(x: any): x is Iterable<T> => { + return isObject(x) && isFunction(x[Symbol.iterator]); +}; + +/** @ignore */ +export const isAsyncIterable = <T = any>(x: any): x is AsyncIterable<T> => { + return isObject(x) && isFunction(x[Symbol.asyncIterator]); +}; + +/** @ignore */ +export const isArrowJSON = (x: any): x is ArrowJSONLike => { + return isObject(x) && isObject(x['schema']); +}; + +/** @ignore */ +export const isArrayLike = <T = any>(x: any): x is ArrayLike<T> => { + return isObject(x) && isNumber(x['length']); +}; + +/** @ignore */ +export const isIteratorResult = <T = any>(x: any): x is IteratorResult<T> => { + return isObject(x) && ('done' in x) && ('value' in x); +}; + +/** @ignore */ +export const isUnderlyingSink = <T = any>(x: any): x is UnderlyingSink<T> => { + return isObject(x) && + isFunction(x['abort']) && + isFunction(x['close']) && + isFunction(x['start']) && + isFunction(x['write']); +}; + +/** @ignore */ +export const isFileHandle = (x: any): x is FileHandle => { + return isObject(x) && isFunction(x['stat']) && isNumber(x['fd']); +}; + +/** @ignore */ +export const isFSReadStream = (x: any): x is FSReadStream => { + return isReadableNodeStream(x) && isNumber((<any> x)['bytesRead']); +}; + +/** @ignore */ +export const isFetchResponse = (x: any): x is Response => { + return isObject(x) && isReadableDOMStream(x['body']); +}; + +/** @ignore */ +export const isWritableDOMStream = <T = any>(x: any): x is WritableStream<T> => { + return isObject(x) && + isFunction(x['abort']) && + isFunction(x['getWriter']) && + !(x instanceof ReadableInterop); +}; + +/** @ignore */ +export const isReadableDOMStream = <T = any>(x: any): x is ReadableStream<T> => { + return isObject(x) && + isFunction(x['cancel']) && + isFunction(x['getReader']) && + !(x instanceof ReadableInterop); +}; + +/** @ignore */ +export const isWritableNodeStream = (x: any): x is NodeJS.WritableStream => { + return isObject(x) && + isFunction(x['end']) && + isFunction(x['write']) && + isBoolean(x['writable']) && + !(x instanceof ReadableInterop); +}; + +/** @ignore */ +export const isReadableNodeStream = (x: any): x is NodeJS.ReadableStream => { + return isObject(x) && + isFunction(x['read']) && + isFunction(x['pipe']) && + isBoolean(x['readable']) && + !(x instanceof ReadableInterop); +}; diff --git a/src/arrow/js/src/util/fn.ts b/src/arrow/js/src/util/fn.ts new file mode 100644 index 000000000..a58f9d337 --- /dev/null +++ b/src/arrow/js/src/util/fn.ts @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/** @ignore */ +export function partial0<T>(visit: (node: T) => any) { + return function(this: T) { return visit(this); }; +} + +/** @ignore */ +export function partial1<T>(visit: (node: T, a: any) => any) { + return function(this: T, a: any) { return visit(this, a); }; +} + +/** @ignore */ +export function partial2<T>(visit: (node: T, a: any, b: any) => any) { + return function(this: T, a: any, b: any) { return visit(this, a, b); }; +} diff --git a/src/arrow/js/src/util/int.ts b/src/arrow/js/src/util/int.ts new file mode 100644 index 000000000..147106dbb --- /dev/null +++ b/src/arrow/js/src/util/int.ts @@ -0,0 +1,440 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/** @ignore */ +const carryBit16 = 1 << 16; + +/** @ignore */ +function intAsHex(value: number): string { + if (value < 0) { + value = 0xFFFFFFFF + value + 1; + } + return `0x${value.toString(16)}`; +} + +/** @ignore */ +const kInt32DecimalDigits = 8; +/** @ignore */ +const kPowersOfTen = [1, + 10, + 100, + 1000, + 10000, + 100000, + 1000000, + 10000000, + 100000000]; + +/** @ignore */ +export class BaseInt64 { + constructor (protected buffer: Uint32Array) {} + + public high(): number { return this.buffer[1]; } + public low (): number { return this.buffer[0]; } + + protected _times(other: BaseInt64) { + // Break the left and right numbers into 16 bit chunks + // so that we can multiply them without overflow. + const L = new Uint32Array([ + this.buffer[1] >>> 16, + this.buffer[1] & 0xFFFF, + this.buffer[0] >>> 16, + this.buffer[0] & 0xFFFF + ]); + + const R = new Uint32Array([ + other.buffer[1] >>> 16, + other.buffer[1] & 0xFFFF, + other.buffer[0] >>> 16, + other.buffer[0] & 0xFFFF + ]); + + let product = L[3] * R[3]; + this.buffer[0] = product & 0xFFFF; + + let sum = product >>> 16; + + product = L[2] * R[3]; + sum += product; + + product = (L[3] * R[2]) >>> 0; + sum += product; + + this.buffer[0] += sum << 16; + + this.buffer[1] = (sum >>> 0 < product ? carryBit16 : 0); + + this.buffer[1] += sum >>> 16; + this.buffer[1] += L[1] * R[3] + L[2] * R[2] + L[3] * R[1]; + this.buffer[1] += (L[0] * R[3] + L[1] * R[2] + L[2] * R[1] + L[3] * R[0]) << 16; + + return this; + } + + protected _plus(other: BaseInt64) { + const sum = (this.buffer[0] + other.buffer[0]) >>> 0; + this.buffer[1] += other.buffer[1]; + if (sum < (this.buffer[0] >>> 0)) { + ++this.buffer[1]; + } + this.buffer[0] = sum; + } + + public lessThan(other: BaseInt64): boolean { + return this.buffer[1] < other.buffer[1] || + (this.buffer[1] === other.buffer[1] && this.buffer[0] < other.buffer[0]); + } + + public equals(other: BaseInt64): boolean { + return this.buffer[1] === other.buffer[1] && this.buffer[0] == other.buffer[0]; + } + + public greaterThan(other: BaseInt64): boolean { + return other.lessThan(this); + } + + public hex(): string { + return `${intAsHex(this.buffer[1])} ${intAsHex(this.buffer[0])}`; + } +} + +/** @ignore */ +export class Uint64 extends BaseInt64 { + public times(other: Uint64): Uint64 { + this._times(other); + return this; + } + + public plus(other: Uint64): Uint64 { + this._plus(other); + return this; + } + + /** @nocollapse */ + public static from(val: any, out_buffer = new Uint32Array(2)): Uint64 { + return Uint64.fromString( + typeof(val) === 'string' ? val : val.toString(), + out_buffer + ); + } + + /** @nocollapse */ + public static fromNumber(num: number, out_buffer = new Uint32Array(2)): Uint64 { + // Always parse numbers as strings - pulling out high and low bits + // directly seems to lose precision sometimes + // For example: + // > -4613034156400212000 >>> 0 + // 721782784 + // The correct lower 32-bits are 721782752 + return Uint64.fromString(num.toString(), out_buffer); + } + + /** @nocollapse */ + public static fromString(str: string, out_buffer = new Uint32Array(2)): Uint64 { + const length = str.length; + + const out = new Uint64(out_buffer); + for (let posn = 0; posn < length;) { + const group = kInt32DecimalDigits < length - posn ? + kInt32DecimalDigits : length - posn; + const chunk = new Uint64(new Uint32Array([parseInt(str.substr(posn, group), 10), 0])); + const multiple = new Uint64(new Uint32Array([kPowersOfTen[group], 0])); + + out.times(multiple); + out.plus(chunk); + + posn += group; + } + + return out; + } + + /** @nocollapse */ + public static convertArray(values: (string|number)[]): Uint32Array { + const data = new Uint32Array(values.length * 2); + for (let i = -1, n = values.length; ++i < n;) { + Uint64.from(values[i], new Uint32Array(data.buffer, data.byteOffset + 2 * i * 4, 2)); + } + return data; + } + + /** @nocollapse */ + public static multiply(left: Uint64, right: Uint64): Uint64 { + const rtrn = new Uint64(new Uint32Array(left.buffer)); + return rtrn.times(right); + } + + /** @nocollapse */ + public static add(left: Uint64, right: Uint64): Uint64 { + const rtrn = new Uint64(new Uint32Array(left.buffer)); + return rtrn.plus(right); + } +} + +/** @ignore */ +export class Int64 extends BaseInt64 { + public negate(): Int64 { + this.buffer[0] = ~this.buffer[0] + 1; + this.buffer[1] = ~this.buffer[1]; + + if (this.buffer[0] == 0) { ++this.buffer[1]; } + return this; + } + + public times(other: Int64): Int64 { + this._times(other); + return this; + } + + public plus(other: Int64): Int64 { + this._plus(other); + return this; + } + + public lessThan(other: Int64): boolean { + // force high bytes to be signed + const this_high = this.buffer[1] << 0; + const other_high = other.buffer[1] << 0; + return this_high < other_high || + (this_high === other_high && this.buffer[0] < other.buffer[0]); + } + + /** @nocollapse */ + public static from(val: any, out_buffer = new Uint32Array(2)): Int64 { + return Int64.fromString( + typeof(val) === 'string' ? val : val.toString(), + out_buffer + ); + } + + /** @nocollapse */ + public static fromNumber(num: number, out_buffer = new Uint32Array(2)): Int64 { + // Always parse numbers as strings - pulling out high and low bits + // directly seems to lose precision sometimes + // For example: + // > -4613034156400212000 >>> 0 + // 721782784 + // The correct lower 32-bits are 721782752 + return Int64.fromString(num.toString(), out_buffer); + } + + /** @nocollapse */ + public static fromString(str: string, out_buffer = new Uint32Array(2)): Int64 { + // TODO: Assert that out_buffer is 0 and length = 2 + const negate = str.startsWith('-'); + const length = str.length; + + const out = new Int64(out_buffer); + for (let posn = negate ? 1 : 0; posn < length;) { + const group = kInt32DecimalDigits < length - posn ? + kInt32DecimalDigits : length - posn; + const chunk = new Int64(new Uint32Array([parseInt(str.substr(posn, group), 10), 0])); + const multiple = new Int64(new Uint32Array([kPowersOfTen[group], 0])); + + out.times(multiple); + out.plus(chunk); + + posn += group; + } + return negate ? out.negate() : out; + } + + /** @nocollapse */ + public static convertArray(values: (string|number)[]): Uint32Array { + const data = new Uint32Array(values.length * 2); + for (let i = -1, n = values.length; ++i < n;) { + Int64.from(values[i], new Uint32Array(data.buffer, data.byteOffset + 2 * i * 4, 2)); + } + return data; + } + + /** @nocollapse */ + public static multiply(left: Int64, right: Int64): Int64 { + const rtrn = new Int64(new Uint32Array(left.buffer)); + return rtrn.times(right); + } + + /** @nocollapse */ + public static add(left: Int64, right: Int64): Int64 { + const rtrn = new Int64(new Uint32Array(left.buffer)); + return rtrn.plus(right); + } +} + +/** @ignore */ +export class Int128 { + constructor (private buffer: Uint32Array) { + // buffer[3] MSB (high) + // buffer[2] + // buffer[1] + // buffer[0] LSB (low) + } + + public high(): Int64 { + return new Int64(new Uint32Array(this.buffer.buffer, this.buffer.byteOffset + 8, 2)); + } + + public low(): Int64 { + return new Int64(new Uint32Array(this.buffer.buffer, this.buffer.byteOffset, 2)); + } + + public negate(): Int128 { + this.buffer[0] = ~this.buffer[0] + 1; + this.buffer[1] = ~this.buffer[1]; + this.buffer[2] = ~this.buffer[2]; + this.buffer[3] = ~this.buffer[3]; + + if (this.buffer[0] == 0) { ++this.buffer[1]; } + if (this.buffer[1] == 0) { ++this.buffer[2]; } + if (this.buffer[2] == 0) { ++this.buffer[3]; } + return this; + } + + public times(other: Int128): Int128 { + // Break the left and right numbers into 32 bit chunks + // so that we can multiply them without overflow. + const L0 = new Uint64(new Uint32Array([this.buffer[3], 0])); + const L1 = new Uint64(new Uint32Array([this.buffer[2], 0])); + const L2 = new Uint64(new Uint32Array([this.buffer[1], 0])); + const L3 = new Uint64(new Uint32Array([this.buffer[0], 0])); + + const R0 = new Uint64(new Uint32Array([other.buffer[3], 0])); + const R1 = new Uint64(new Uint32Array([other.buffer[2], 0])); + const R2 = new Uint64(new Uint32Array([other.buffer[1], 0])); + const R3 = new Uint64(new Uint32Array([other.buffer[0], 0])); + + let product = Uint64.multiply(L3, R3); + this.buffer[0] = product.low(); + + const sum = new Uint64(new Uint32Array([product.high(), 0])); + + product = Uint64.multiply(L2, R3); + sum.plus(product); + + product = Uint64.multiply(L3, R2); + sum.plus(product); + + this.buffer[1] = sum.low(); + + this.buffer[3] = (sum.lessThan(product) ? 1 : 0); + + this.buffer[2] = sum.high(); + const high = new Uint64(new Uint32Array(this.buffer.buffer, this.buffer.byteOffset + 8, 2)); + + high.plus(Uint64.multiply(L1, R3)) + .plus(Uint64.multiply(L2, R2)) + .plus(Uint64.multiply(L3, R1)); + this.buffer[3] += Uint64.multiply(L0, R3) + .plus(Uint64.multiply(L1, R2)) + .plus(Uint64.multiply(L2, R1)) + .plus(Uint64.multiply(L3, R0)).low(); + + return this; + } + + public plus(other: Int128): Int128 { + const sums = new Uint32Array(4); + sums[3] = (this.buffer[3] + other.buffer[3]) >>> 0; + sums[2] = (this.buffer[2] + other.buffer[2]) >>> 0; + sums[1] = (this.buffer[1] + other.buffer[1]) >>> 0; + sums[0] = (this.buffer[0] + other.buffer[0]) >>> 0; + + if (sums[0] < (this.buffer[0] >>> 0)) { + ++sums[1]; + } + if (sums[1] < (this.buffer[1] >>> 0)) { + ++sums[2]; + } + if (sums[2] < (this.buffer[2] >>> 0)) { + ++sums[3]; + } + + this.buffer[3] = sums[3]; + this.buffer[2] = sums[2]; + this.buffer[1] = sums[1]; + this.buffer[0] = sums[0]; + + return this; + } + + public hex(): string { + return `${intAsHex(this.buffer[3])} ${intAsHex(this.buffer[2])} ${intAsHex(this.buffer[1])} ${intAsHex(this.buffer[0])}`; + } + + /** @nocollapse */ + public static multiply(left: Int128, right: Int128): Int128 { + const rtrn = new Int128(new Uint32Array(left.buffer)); + return rtrn.times(right); + } + + /** @nocollapse */ + public static add(left: Int128, right: Int128): Int128 { + const rtrn = new Int128(new Uint32Array(left.buffer)); + return rtrn.plus(right); + } + + /** @nocollapse */ + public static from(val: any, out_buffer = new Uint32Array(4)): Int128 { + return Int128.fromString( + typeof(val) === 'string' ? val : val.toString(), + out_buffer + ); + } + + /** @nocollapse */ + public static fromNumber(num: number, out_buffer = new Uint32Array(4)): Int128 { + // Always parse numbers as strings - pulling out high and low bits + // directly seems to lose precision sometimes + // For example: + // > -4613034156400212000 >>> 0 + // 721782784 + // The correct lower 32-bits are 721782752 + return Int128.fromString(num.toString(), out_buffer); + } + + /** @nocollapse */ + public static fromString(str: string, out_buffer = new Uint32Array(4)): Int128 { + // TODO: Assert that out_buffer is 0 and length = 4 + const negate = str.startsWith('-'); + const length = str.length; + + const out = new Int128(out_buffer); + for (let posn = negate ? 1 : 0; posn < length;) { + const group = kInt32DecimalDigits < length - posn ? + kInt32DecimalDigits : length - posn; + const chunk = new Int128(new Uint32Array([parseInt(str.substr(posn, group), 10), 0, 0, 0])); + const multiple = new Int128(new Uint32Array([kPowersOfTen[group], 0, 0, 0])); + + out.times(multiple); + out.plus(chunk); + + posn += group; + } + + return negate ? out.negate() : out; + } + + /** @nocollapse */ + public static convertArray(values: (string|number)[]): Uint32Array { + // TODO: Distinguish between string and number at compile-time + const data = new Uint32Array(values.length * 4); + for (let i = -1, n = values.length; ++i < n;) { + Int128.from(values[i], new Uint32Array(data.buffer, data.byteOffset + 4 * 4 * i, 4)); + } + return data; + } +} diff --git a/src/arrow/js/src/util/math.ts b/src/arrow/js/src/util/math.ts new file mode 100644 index 000000000..47678e1a9 --- /dev/null +++ b/src/arrow/js/src/util/math.ts @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const f64 = new Float64Array(1); +const u32 = new Uint32Array(f64.buffer); + +/** + * Convert uint16 (logically a float16) to a JS float64. Inspired by numpy's `npy_half_to_double`: + * https://github.com/numpy/numpy/blob/5a5987291dc95376bb098be8d8e5391e89e77a2c/numpy/core/src/npymath/halffloat.c#L29 + * @param h {number} the uint16 to convert + * @private + * @ignore + */ +export function uint16ToFloat64(h: number) { + const expo = (h & 0x7C00) >> 10; + const sigf = (h & 0x03FF) / 1024; + const sign = (-1) ** ((h & 0x8000) >> 15); + switch (expo) { + case 0x1F: return sign * (sigf ? NaN : 1 / 0); + case 0x00: return sign * (sigf ? 6.103515625e-5 * sigf : 0); + } + return sign * (2 ** (expo - 15)) * (1 + sigf); +} + +/** + * Convert a float64 to uint16 (assuming the float64 is logically a float16). Inspired by numpy's `npy_double_to_half`: + * https://github.com/numpy/numpy/blob/5a5987291dc95376bb098be8d8e5391e89e77a2c/numpy/core/src/npymath/halffloat.c#L43 + * @param d {number} The float64 to convert + * @private + * @ignore + */ +export function float64ToUint16(d: number) { + + if (d !== d) { return 0x7E00; } // NaN + + f64[0] = d; + + // Magic numbers: + // 0x80000000 = 10000000 00000000 00000000 00000000 -- masks the 32nd bit + // 0x7ff00000 = 01111111 11110000 00000000 00000000 -- masks the 21st-31st bits + // 0x000fffff = 00000000 00001111 11111111 11111111 -- masks the 1st-20th bit + + const sign = (u32[1] & 0x80000000) >> 16 & 0xFFFF; + let expo = (u32[1] & 0x7ff00000), sigf = 0x0000; + + if (expo >= 0x40f00000) { + // + // If exponent overflowed, the float16 is either NaN or Infinity. + // Rules to propagate the sign bit: mantissa > 0 ? NaN : +/-Infinity + // + // Magic numbers: + // 0x40F00000 = 01000000 11110000 00000000 00000000 -- 6-bit exponent overflow + // 0x7C000000 = 01111100 00000000 00000000 00000000 -- masks the 27th-31st bits + // + // returns: + // qNaN, aka 32256 decimal, 0x7E00 hex, or 01111110 00000000 binary + // sNaN, aka 32000 decimal, 0x7D00 hex, or 01111101 00000000 binary + // +inf, aka 31744 decimal, 0x7C00 hex, or 01111100 00000000 binary + // -inf, aka 64512 decimal, 0xFC00 hex, or 11111100 00000000 binary + // + // If mantissa is greater than 23 bits, set to +Infinity like numpy + if (u32[0] > 0) { + expo = 0x7C00; + } else { + expo = (expo & 0x7C000000) >> 16; + sigf = (u32[1] & 0x000fffff) >> 10; + } + } else if (expo <= 0x3f000000) { + // + // If exponent underflowed, the float is either signed zero or subnormal. + // + // Magic numbers: + // 0x3F000000 = 00111111 00000000 00000000 00000000 -- 6-bit exponent underflow + // + sigf = 0x100000 + (u32[1] & 0x000fffff); + sigf = 0x100000 + (sigf << ((expo >> 20) - 998)) >> 21; + expo = 0; + } else { + // + // No overflow or underflow, rebase the exponent and round the mantissa + // Magic numbers: + // 0x200 = 00000010 00000000 -- masks off the 10th bit + // + + // Ensure the first mantissa bit (the 10th one) is 1 and round + expo = (expo - 0x3f000000) >> 10; + sigf = ((u32[1] & 0x000fffff) + 0x200) >> 10; + } + + return sign | expo | sigf & 0xFFFF; +} diff --git a/src/arrow/js/src/util/pretty.ts b/src/arrow/js/src/util/pretty.ts new file mode 100644 index 000000000..a189fc490 --- /dev/null +++ b/src/arrow/js/src/util/pretty.ts @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/** @ignore */ const undf = void (0); + +/** @ignore */ +export function valueToString(x: any) { + if (x === null) { return 'null'; } + if (x === undf) { return 'undefined'; } + switch (typeof x) { + case 'number': return `${x}`; + case 'bigint': return `${x}`; + case 'string': return `"${x}"`; + } + // If [Symbol.toPrimitive] is implemented (like in BN) + // use it instead of JSON.stringify(). This ensures we + // print BigInts, Decimals, and Binary in their native + // representation + if (typeof x[Symbol.toPrimitive] === 'function') { + return x[Symbol.toPrimitive]('string'); + } + return ArrayBuffer.isView(x) ? `[${x}]` : JSON.stringify(x); +} diff --git a/src/arrow/js/src/util/recordbatch.ts b/src/arrow/js/src/util/recordbatch.ts new file mode 100644 index 000000000..37a630858 --- /dev/null +++ b/src/arrow/js/src/util/recordbatch.ts @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Column } from '../column'; +import { Vector } from '../vector'; +import { DataType } from '../type'; +import { Data, Buffers } from '../data'; +import { Schema, Field } from '../schema'; +import { Chunked } from '../vector/chunked'; +import { RecordBatch } from '../recordbatch'; + +const noopBuf = new Uint8Array(0); +const nullBufs = (bitmapLength: number) => <unknown> [ + noopBuf, noopBuf, new Uint8Array(bitmapLength), noopBuf +] as Buffers<any>; + +/** @ignore */ +export function ensureSameLengthData<T extends { [key: string]: DataType } = any>( + schema: Schema<T>, + chunks: Data<T[keyof T]>[], + batchLength = chunks.reduce((l, c) => Math.max(l, c.length), 0) +) { + let data: Data<T[keyof T]>; + let field: Field<T[keyof T]>; + let i = -1; + const n = chunks.length; + const fields = [...schema.fields]; + const batchData = [] as Data<T[keyof T]>[]; + const bitmapLength = ((batchLength + 63) & ~63) >> 3; + while (++i < n) { + if ((data = chunks[i]) && data.length === batchLength) { + batchData[i] = data; + } else { + (field = fields[i]).nullable || (fields[i] = fields[i].clone({ nullable: true }) as Field<T[keyof T]>); + batchData[i] = data ? data._changeLengthAndBackfillNullBitmap(batchLength) + : Data.new(field.type, 0, batchLength, batchLength, nullBufs(bitmapLength)) as Data<T[keyof T]>; + } + } + return [new Schema<T>(fields), batchLength, batchData] as [Schema<T>, number, Data<T[keyof T]>[]]; +} + +/** @ignore */ +export function distributeColumnsIntoRecordBatches<T extends { [key: string]: DataType } = any>(columns: Column<T[keyof T]>[]): [Schema<T>, RecordBatch<T>[]] { + return distributeVectorsIntoRecordBatches<T>(new Schema<T>(columns.map(({ field }) => field)), columns); +} + +/** @ignore */ +export function distributeVectorsIntoRecordBatches<T extends { [key: string]: DataType } = any>(schema: Schema<T>, vecs: (Vector<T[keyof T]> | Chunked<T[keyof T]>)[]): [Schema<T>, RecordBatch<T>[]] { + return uniformlyDistributeChunksAcrossRecordBatches<T>(schema, vecs.map((v) => v instanceof Chunked ? v.chunks.map((c) => c.data) : [v.data])); +} + +/** @ignore */ +function uniformlyDistributeChunksAcrossRecordBatches<T extends { [key: string]: DataType } = any>(schema: Schema<T>, columns: Data<T[keyof T]>[][]): [Schema<T>, RecordBatch<T>[]] { + + const fields = [...schema.fields]; + const batchArgs = [] as [number, Data<T[keyof T]>[]][]; + const memo = { numBatches: columns.reduce((n, c) => Math.max(n, c.length), 0) }; + + let numBatches = 0, batchLength = 0; + let i = -1; + const numColumns = columns.length; + let child: Data<T[keyof T]>, childData: Data<T[keyof T]>[] = []; + + while (memo.numBatches-- > 0) { + + for (batchLength = Number.POSITIVE_INFINITY, i = -1; ++i < numColumns;) { + childData[i] = child = columns[i].shift()!; + batchLength = Math.min(batchLength, child ? child.length : batchLength); + } + + if (isFinite(batchLength)) { + childData = distributeChildData(fields, batchLength, childData, columns, memo); + if (batchLength > 0) { + batchArgs[numBatches++] = [batchLength, childData.slice()]; + } + } + } + return [ + schema = new Schema<T>(fields, schema.metadata), + batchArgs.map((xs) => new RecordBatch(schema, ...xs)) + ]; +} + +/** @ignore */ +function distributeChildData<T extends { [key: string]: DataType } = any>(fields: Field<T[keyof T]>[], batchLength: number, childData: Data<T[keyof T]>[], columns: Data<T[keyof T]>[][], memo: { numBatches: number }) { + let data: Data<T[keyof T]>; + let field: Field<T[keyof T]>; + let length = 0, i = -1; + const n = columns.length; + const bitmapLength = ((batchLength + 63) & ~63) >> 3; + while (++i < n) { + if ((data = childData[i]) && ((length = data.length) >= batchLength)) { + if (length === batchLength) { + childData[i] = data; + } else { + childData[i] = data.slice(0, batchLength); + data = data.slice(batchLength, length - batchLength); + memo.numBatches = Math.max(memo.numBatches, columns[i].unshift(data)); + } + } else { + (field = fields[i]).nullable || (fields[i] = field.clone({ nullable: true }) as Field<T[keyof T]>); + childData[i] = data ? data._changeLengthAndBackfillNullBitmap(batchLength) + : Data.new(field.type, 0, batchLength, batchLength, nullBufs(bitmapLength)) as Data<T[keyof T]>; + } + } + return childData; +} diff --git a/src/arrow/js/src/util/utf8.ts b/src/arrow/js/src/util/utf8.ts new file mode 100644 index 000000000..b6f8fcdb8 --- /dev/null +++ b/src/arrow/js/src/util/utf8.ts @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const decoder = new TextDecoder('utf-8'); +/** @ignore */ +export const decodeUtf8 = (buffer?: BufferSource) => decoder.decode(buffer); + +const encoder = new TextEncoder(); +/** @ignore */ +export const encodeUtf8 = (value?: string) => encoder.encode(value); diff --git a/src/arrow/js/src/util/vector.ts b/src/arrow/js/src/util/vector.ts new file mode 100644 index 000000000..a6cfd0373 --- /dev/null +++ b/src/arrow/js/src/util/vector.ts @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from '../vector'; +import { MapRow, StructRow } from '../vector/row'; +import { compareArrayLike } from '../util/buffer'; +import { BigInt, BigIntAvailable } from './compat'; + +/** @ignore */ +type RangeLike = { length: number; stride?: number }; +/** @ignore */ +type ClampThen<T extends RangeLike> = (source: T, index: number) => any; +/** @ignore */ +type ClampRangeThen<T extends RangeLike> = (source: T, offset: number, length: number) => any; + +export function clampIndex<T extends RangeLike>(source: T, index: number): number; +export function clampIndex<T extends RangeLike, N extends ClampThen<T> = ClampThen<T>>(source: T, index: number, then: N): ReturnType<N>; +/** @ignore */ +export function clampIndex<T extends RangeLike, N extends ClampThen<T> = ClampThen<T>>(source: T, index: number, then?: N) { + const length = source.length; + const adjust = index > -1 ? index : (length + (index % length)); + return then ? then(source, adjust) : adjust; +} + +/** @ignore */ +let tmp: number; +export function clampRange<T extends RangeLike>(source: T, begin: number | undefined, end: number | undefined): [number, number]; +export function clampRange<T extends RangeLike, N extends ClampRangeThen<T> = ClampRangeThen<T>>(source: T, begin: number | undefined, end: number | undefined, then: N): ReturnType<N>; +/** @ignore */ +export function clampRange<T extends RangeLike, N extends ClampRangeThen<T> = ClampRangeThen<T>>(source: T, begin: number | undefined, end: number | undefined, then?: N) { + + // Adjust args similar to Array.prototype.slice. Normalize begin/end to + // clamp between 0 and length, and wrap around on negative indices, e.g. + // slice(-1, 5) or slice(5, -1) + const { length: len = 0 } = source; + let lhs = typeof begin !== 'number' ? 0 : begin; + let rhs = typeof end !== 'number' ? len : end; + // wrap around on negative start/end positions + (lhs < 0) && (lhs = ((lhs % len) + len) % len); + (rhs < 0) && (rhs = ((rhs % len) + len) % len); + // ensure lhs <= rhs + (rhs < lhs) && (tmp = lhs, lhs = rhs, rhs = tmp); + // ensure rhs <= length + (rhs > len) && (rhs = len); + + return then ? then(source, lhs, rhs) : [lhs, rhs]; +} + +const big0 = BigIntAvailable ? BigInt(0) : 0; +const isNaNFast = (value: any) => value !== value; + +/** @ignore */ +export function createElementComparator(search: any) { + const typeofSearch = typeof search; + // Compare primitives + if (typeofSearch !== 'object' || search === null) { + // Compare NaN + if (isNaNFast(search)) { + return isNaNFast; + } + return typeofSearch !== 'bigint' + ? (value: any) => value === search + : (value: any) => (big0 + value) === search; + } + // Compare Dates + if (search instanceof Date) { + const valueOfSearch = search.valueOf(); + return (value: any) => value instanceof Date ? (value.valueOf() === valueOfSearch) : false; + } + // Compare TypedArrays + if (ArrayBuffer.isView(search)) { + return (value: any) => value ? compareArrayLike(search, value) : false; + } + // Compare Maps and Rows + if (search instanceof Map) { return creatMapComparator(search); } + // Compare Array-likes + if (Array.isArray(search)) { return createArrayLikeComparator(search); } + // Compare Vectors + if (search instanceof Vector) { return createVectorComparator(search); } + // Compare non-empty Objects + return createObjectComparator(search); +} + +/** @ignore */ +function createArrayLikeComparator(lhs: ArrayLike<any>) { + const comparators = [] as ((x: any) => boolean)[]; + for (let i = -1, n = lhs.length; ++i < n;) { + comparators[i] = createElementComparator(lhs[i]); + } + return createSubElementsComparator(comparators); +} + +/** @ignore */ +function creatMapComparator(lhs: Map<any, any>) { + let i = -1; + const comparators = [] as ((x: any) => boolean)[]; + lhs.forEach((v) => comparators[++i] = createElementComparator(v)); + return createSubElementsComparator(comparators); +} + +/** @ignore */ +function createVectorComparator(lhs: Vector<any>) { + const comparators = [] as ((x: any) => boolean)[]; + for (let i = -1, n = lhs.length; ++i < n;) { + comparators[i] = createElementComparator(lhs.get(i)); + } + return createSubElementsComparator(comparators); +} + +/** @ignore */ +function createObjectComparator(lhs: any) { + const keys = Object.keys(lhs); + // Only compare non-empty Objects + if (keys.length === 0) { return () => false; } + const comparators = [] as ((x: any) => boolean)[]; + for (let i = -1, n = keys.length; ++i < n;) { + comparators[i] = createElementComparator(lhs[keys[i]]); + } + return createSubElementsComparator(comparators, keys); +} + +function createSubElementsComparator(comparators: ((x: any) => boolean)[], keys?: Iterable<string>) { + return (rhs: any) => { + if (!rhs || typeof rhs !== 'object') { + return false; + } + switch (rhs.constructor) { + case Array: return compareArray(comparators, rhs); + case Map: + case MapRow: + case StructRow: + return compareObject(comparators, rhs, rhs.keys()); + case Object: + case undefined: // support `Object.create(null)` objects + return compareObject(comparators, rhs, keys || Object.keys(rhs)); + } + return rhs instanceof Vector ? compareVector(comparators, rhs) : false; + }; +} + +function compareArray(comparators: ((x: any) => boolean)[], arr: any[]) { + const n = comparators.length; + if (arr.length !== n) { return false; } + for (let i = -1; ++i < n;) { + if (!(comparators[i](arr[i]))) { return false; } + } + return true; +} + +function compareVector(comparators: ((x: any) => boolean)[], vec: Vector) { + const n = comparators.length; + if (vec.length !== n) { return false; } + for (let i = -1; ++i < n;) { + if (!(comparators[i](vec.get(i)))) { return false; } + } + return true; +} + +function compareObject(comparators: ((x: any) => boolean)[], obj: Map<any, any>, keys: Iterable<string>) { + + const lKeyItr = keys[Symbol.iterator](); + const rKeyItr = obj instanceof Map ? obj.keys() : Object.keys(obj)[Symbol.iterator](); + const rValItr = obj instanceof Map ? obj.values() : Object.values(obj)[Symbol.iterator](); + + let i = 0; + const n = comparators.length; + let rVal = rValItr.next(); + let lKey = lKeyItr.next(); + let rKey = rKeyItr.next(); + + for (; i < n && !lKey.done && !rKey.done && !rVal.done; + ++i, lKey = lKeyItr.next(), rKey = rKeyItr.next(), rVal = rValItr.next()) { + if (lKey.value !== rKey.value || !comparators[i](rVal.value)) { + break; + } + } + if (i === n && lKey.done && rKey.done && rVal.done) { + return true; + } + lKeyItr.return && lKeyItr.return(); + rKeyItr.return && rKeyItr.return(); + rValItr.return && rValItr.return(); + return false; +} diff --git a/src/arrow/js/src/vector.ts b/src/arrow/js/src/vector.ts new file mode 100644 index 000000000..bd7838cdf --- /dev/null +++ b/src/arrow/js/src/vector.ts @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from './data'; +import { DataType } from './type'; +import { Chunked } from './vector/chunked'; + +/** @ignore */ +export interface Clonable<R extends AbstractVector> { + clone(...args: any[]): R; +} + +/** @ignore */ +export interface Sliceable<R extends AbstractVector> { + slice(begin?: number, end?: number): R; +} + +/** @ignore */ +export interface Applicative<T extends DataType, R extends Chunked> { + concat(...others: Vector<T>[]): R; + readonly [Symbol.isConcatSpreadable]: boolean; +} + +export interface AbstractVector<T extends DataType = any> + extends Clonable<AbstractVector<T>>, + Sliceable<AbstractVector<T>>, + Applicative<T, Chunked<T>> { + + readonly TType: T['TType']; + readonly TArray: T['TArray']; + readonly TValue: T['TValue']; +} + +export abstract class AbstractVector<T extends DataType = any> implements Iterable<T['TValue'] | null> { + + public abstract readonly data: Data<T>; + public abstract readonly type: T; + public abstract readonly typeId: T['TType']; + public abstract readonly length: number; + public abstract readonly stride: number; + public abstract readonly nullCount: number; + public abstract readonly byteLength: number; + public abstract readonly numChildren: number; + + public abstract readonly ArrayType: T['ArrayType']; + + public abstract isValid(index: number): boolean; + public abstract get(index: number): T['TValue'] | null; + public abstract set(index: number, value: T['TValue'] | null): void; + public abstract indexOf(value: T['TValue'] | null, fromIndex?: number): number; + public abstract [Symbol.iterator](): IterableIterator<T['TValue'] | null>; + + public abstract toArray(): T['TArray']; + public abstract getChildAt<R extends DataType = any>(index: number): Vector<R> | null; +} + +(AbstractVector.prototype as any).data = null; + +export { AbstractVector as Vector }; diff --git a/src/arrow/js/src/vector/base.ts b/src/arrow/js/src/vector/base.ts new file mode 100644 index 000000000..2ceecdda4 --- /dev/null +++ b/src/arrow/js/src/vector/base.ts @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import { Type } from '../enum'; +import { DataType } from '../type'; +import { Chunked } from './chunked'; +import { clampRange } from '../util/vector'; +import { VectorType as V } from '../interfaces'; +import { AbstractVector, Vector, Clonable, Sliceable, Applicative } from '../vector'; + +/** @ignore */ +export interface BaseVector<T extends DataType = any> extends Clonable<V<T>>, Sliceable<V<T>>, Applicative<T, Chunked<T>> { + slice(begin?: number, end?: number): V<T>; + concat(...others: Vector<T>[]): Chunked<T>; + clone<R extends DataType = T>(data: Data<R>, children?: Vector<R>[]): V<R>; +} + +/** @ignore */ +export abstract class BaseVector<T extends DataType = any> extends AbstractVector<T> + implements Clonable<V<T>>, Sliceable<V<T>>, Applicative<T, Chunked<T>> { + + protected _children?: Vector[]; + + constructor(data: Data<T>, children?: Vector[]) { + super(); + this._children = children; + this.numChildren = data.childData.length; + this._bindDataAccessors(this.data = data); + } + + public readonly data: Data<T>; + public readonly numChildren: number; + + public get type() { return this.data.type; } + public get typeId() { return this.data.typeId; } + public get length() { return this.data.length; } + public get offset() { return this.data.offset; } + public get stride() { return this.data.stride; } + public get nullCount() { return this.data.nullCount; } + public get byteLength() { return this.data.byteLength; } + public get VectorName() { return `${Type[this.typeId]}Vector`; } + + public get ArrayType(): T['ArrayType'] { return this.type.ArrayType; } + + public get values() { return this.data.values; } + public get typeIds() { return this.data.typeIds; } + public get nullBitmap() { return this.data.nullBitmap; } + public get valueOffsets() { return this.data.valueOffsets; } + + public get [Symbol.toStringTag]() { return `${this.VectorName}<${this.type[Symbol.toStringTag]}>`; } + + public clone<R extends DataType = T>(data: Data<R>, children = this._children) { + return Vector.new<R>(data, children) as any; + } + + public concat(...others: Vector<T>[]) { + return Chunked.concat<T>(this, ...others); + } + + public slice(begin?: number, end?: number) { + // Adjust args similar to Array.prototype.slice. Normalize begin/end to + // clamp between 0 and length, and wrap around on negative indices, e.g. + // slice(-1, 5) or slice(5, -1) + return clampRange(this, begin, end, this._sliceInternal); + } + + public isValid(index: number): boolean { + if (this.nullCount > 0) { + const idx = this.offset + index; + const val = this.nullBitmap[idx >> 3]; + const mask = (val & (1 << (idx % 8))); + return mask !== 0; + } + return true; + } + + public getChildAt<R extends DataType = any>(index: number): Vector<R> | null { + return index < 0 || index >= this.numChildren ? null : ( + (this._children || (this._children = []))[index] || + (this._children[index] = Vector.new<R>(this.data.childData[index] as Data<R>)) + ) as Vector<R>; + } + + public toJSON() { return [...this]; } + + protected _sliceInternal(self: this, begin: number, end: number) { + return self.clone(self.data.slice(begin, end - begin), null!); + } + + // @ts-ignore + protected _bindDataAccessors(data: Data<T>) { + // Implementation in src/vectors/index.ts due to circular dependency/packaging shenanigans + } +} + +(BaseVector.prototype as any)[Symbol.isConcatSpreadable] = true; diff --git a/src/arrow/js/src/vector/binary.ts b/src/arrow/js/src/vector/binary.ts new file mode 100644 index 000000000..603187a78 --- /dev/null +++ b/src/arrow/js/src/vector/binary.ts @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from '../vector'; +import { BaseVector } from './base'; +import { Binary, Utf8 } from '../type'; + +/** @ignore */ +export class BinaryVector extends BaseVector<Binary> { + public asUtf8() { + return Vector.new(this.data.clone(new Utf8())); + } +} diff --git a/src/arrow/js/src/vector/bool.ts b/src/arrow/js/src/vector/bool.ts new file mode 100644 index 000000000..b555f4692 --- /dev/null +++ b/src/arrow/js/src/vector/bool.ts @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Bool } from '../type'; +import { Chunked } from './chunked'; +import { BaseVector } from './base'; +import { VectorBuilderOptions } from './index'; +import { vectorFromValuesWithType } from './index'; +import { VectorBuilderOptionsAsync } from './index'; + +/** @ignore */ +export class BoolVector extends BaseVector<Bool> { + public static from<TNull = any>(input: Iterable<boolean | TNull>): BoolVector; + public static from<TNull = any>(input: AsyncIterable<boolean | TNull>): Promise<BoolVector>; + public static from<TNull = any>(input: VectorBuilderOptions<Bool, boolean | TNull>): Chunked<Bool>; + public static from<TNull = any>(input: VectorBuilderOptionsAsync<Bool, boolean | TNull>): Promise<Chunked<Bool>>; + /** @nocollapse */ + public static from<TNull = any>(input: Iterable<boolean | TNull> | AsyncIterable<boolean | TNull> | VectorBuilderOptions<Bool, boolean | TNull> | VectorBuilderOptionsAsync<Bool, boolean | TNull>) { + return vectorFromValuesWithType(() => new Bool(), input); + } +} diff --git a/src/arrow/js/src/vector/chunked.ts b/src/arrow/js/src/vector/chunked.ts new file mode 100644 index 000000000..656c4a1b6 --- /dev/null +++ b/src/arrow/js/src/vector/chunked.ts @@ -0,0 +1,320 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import { Field } from '../schema'; +import { clampRange } from '../util/vector'; +import { DataType, Dictionary } from '../type'; +import { selectChunkArgs } from '../util/args'; +import { DictionaryVector } from './dictionary'; +import { AbstractVector, Vector } from '../vector'; +import { Clonable, Sliceable, Applicative } from '../vector'; + +/** @ignore */ +type ChunkedDict<T extends DataType> = T extends Dictionary ? Vector<T['dictionary']> : null | never; +/** @ignore */ +type ChunkedKeys<T extends DataType> = T extends Dictionary ? Vector<T['indices']> | Chunked<T['indices']> : null | never; + +/** @ignore */ +export type SearchContinuation<T extends Chunked> = (column: T, chunkIndex: number, valueIndex: number) => any; + +/** @ignore */ +class ChunkedIterator<T extends DataType> implements IterableIterator<T['TValue'] | null> { + private chunkIndex = 0; + private chunkIterator: IterableIterator<T['TValue'] | null>; + + constructor( + private chunks: Vector<T>[], + ) { + this.chunkIterator = this.getChunkIterator(); + } + + next(): IteratorResult<T['TValue'] | null> { + while (this.chunkIndex < this.chunks.length) { + const next = this.chunkIterator.next(); + + if (!next.done) { + return next; + } + + if (++this.chunkIndex < this.chunks.length) { + this.chunkIterator = this.getChunkIterator(); + } + } + + return {done: true, value: null}; + } + + getChunkIterator() { + return this.chunks[this.chunkIndex][Symbol.iterator](); + } + + [Symbol.iterator]() { + return this; + } +} + +/** @ignore */ +export class Chunked<T extends DataType = any> + extends AbstractVector<T> + implements Clonable<Chunked<T>>, + Sliceable<Chunked<T>>, + Applicative<T, Chunked<T>> { + + /** @nocollapse */ + public static flatten<T extends DataType>(...vectors: (Vector<T> | Vector<T>[])[]) { + return selectChunkArgs<Vector<T>>(Vector, vectors); + } + + /** @nocollapse */ + public static concat<T extends DataType>(...vectors: (Vector<T> | Vector<T>[])[]) { + const chunks = Chunked.flatten<T>(...vectors); + return new Chunked<T>(chunks[0].type, chunks); + } + + protected _type: T; + protected _length: number; + protected _chunks: Vector<T>[]; + protected _numChildren: number; + protected _children?: Chunked[]; + protected _nullCount = -1; + protected _chunkOffsets: Uint32Array; + + constructor(type: T, chunks: Vector<T>[] = [], offsets = calculateOffsets(chunks)) { + super(); + this._type = type; + this._chunks = chunks; + this._chunkOffsets = offsets; + this._length = offsets[offsets.length - 1]; + this._numChildren = (this._type.children || []).length; + } + + public get type() { return this._type; } + public get length() { return this._length; } + public get chunks() { return this._chunks; } + public get typeId(): T['TType'] { return this._type.typeId; } + public get VectorName() { return `Chunked<${this._type}>`; } + public get data(): Data<T> { + return this._chunks[0] ? this._chunks[0].data : <any> null; + } + + public get ArrayType() { return this._type.ArrayType; } + public get numChildren() { return this._numChildren; } + public get stride() { return this._chunks[0] ? this._chunks[0].stride : 1; } + public get byteLength(): number { + return this._chunks.reduce((byteLength, chunk) => byteLength + chunk.byteLength, 0); + } + public get nullCount() { + let nullCount = this._nullCount; + if (nullCount < 0) { + this._nullCount = nullCount = this._chunks.reduce((x, { nullCount }) => x + nullCount, 0); + } + return nullCount; + } + + protected _indices?: ChunkedKeys<T>; + public get indices(): ChunkedKeys<T> | null { + if (DataType.isDictionary(this._type)) { + if (!this._indices) { + const chunks = (<any> this._chunks) as DictionaryVector<T, any>[]; + this._indices = (chunks.length === 1 + ? chunks[0].indices + : Chunked.concat(...chunks.map((x) => x.indices))) as ChunkedKeys<T>; + } + return this._indices; + } + return null; + } + public get dictionary(): ChunkedDict<T> | null { + if (DataType.isDictionary(this._type)) { + return this._chunks[this._chunks.length - 1].data.dictionary as ChunkedDict<T>; + } + return null; + } + + public [Symbol.iterator](): IterableIterator<T['TValue'] | null> { + return new ChunkedIterator(this._chunks); + } + + public clone(chunks = this._chunks): Chunked<T> { + return new Chunked(this._type, chunks); + } + + public concat(...others: Vector<T>[]): Chunked<T> { + return this.clone(Chunked.flatten(this, ...others)); + } + + public slice(begin?: number, end?: number): Chunked<T> { + return clampRange(this, begin, end, this._sliceInternal); + } + + public getChildAt<R extends DataType = any>(index: number): Chunked<R> | null { + + if (index < 0 || index >= this._numChildren) { return null; } + + const columns = this._children || (this._children = []); + let child: Chunked<R>, field: Field<R>, chunks: Vector<R>[]; + + if (child = columns[index]) { return child; } + if (field = ((this._type.children || [])[index] as Field<R>)) { + chunks = this._chunks + .map((vector) => vector.getChildAt<R>(index)) + .filter((vec): vec is Vector<R> => vec != null); + if (chunks.length > 0) { + return (columns[index] = new Chunked<R>(field.type, chunks)); + } + } + + return null; + } + + public search(index: number): [number, number] | null; + public search<N extends SearchContinuation<Chunked<T>>>(index: number, then?: N): ReturnType<N>; + public search<N extends SearchContinuation<Chunked<T>>>(index: number, then?: N) { + const idx = index; + // binary search to find the child vector and value indices + const offsets = this._chunkOffsets; + let rhs = offsets.length - 1; + // return early if out of bounds, or if there's just one child + if (idx < 0 ) { return null; } + if (idx >= offsets[rhs]) { return null; } + if (rhs <= 1 ) { return then ? then(this, 0, idx) : [0, idx]; } + let lhs = 0, pos = 0, mid = 0; + do { + if (lhs + 1 === rhs) { + return then ? then(this, lhs, idx - pos) : [lhs, idx - pos]; + } + mid = lhs + ((rhs - lhs) / 2) | 0; + idx >= offsets[mid] ? (lhs = mid) : (rhs = mid); + } while (idx < offsets[rhs] && idx >= (pos = offsets[lhs])); + return null; + } + + public isValid(index: number): boolean { + return !!this.search(index, this.isValidInternal); + } + + public get(index: number): T['TValue'] | null { + return this.search(index, this.getInternal); + } + + public set(index: number, value: T['TValue'] | null): void { + this.search(index, ({ chunks }, i, j) => chunks[i].set(j, value)); + } + + public indexOf(element: T['TValue'], offset?: number): number { + if (offset && typeof offset === 'number') { + return this.search(offset, (self, i, j) => this.indexOfInternal(self, i, j, element))!; + } + return this.indexOfInternal(this, 0, Math.max(0, offset || 0), element); + } + + public toArray(): T['TArray'] { + const { chunks } = this; + const n = chunks.length; + let ArrayType: any = this._type.ArrayType; + if (n <= 0) { return new ArrayType(0); } + if (n <= 1) { return chunks[0].toArray(); } + let len = 0; + const src = new Array(n); + for (let i = -1; ++i < n;) { + len += (src[i] = chunks[i].toArray()).length; + } + if (ArrayType !== src[0].constructor) { + ArrayType = src[0].constructor; + } + const dst = new ArrayType(len); + const set: any = ArrayType === Array ? arraySet : typedSet; + for (let i = -1, idx = 0; ++i < n;) { + idx = set(src[i], dst, idx); + } + return dst; + } + + protected getInternal({ _chunks }: Chunked<T>, i: number, j: number) { return _chunks[i].get(j); } + protected isValidInternal({ _chunks }: Chunked<T>, i: number, j: number) { return _chunks[i].isValid(j); } + protected indexOfInternal({ _chunks }: Chunked<T>, chunkIndex: number, fromIndex: number, element: T['TValue']) { + let i = chunkIndex - 1; + const n = _chunks.length; + let start = fromIndex, offset = 0, found = -1; + while (++i < n) { + if (~(found = _chunks[i].indexOf(element, start))) { + return offset + found; + } + start = 0; + offset += _chunks[i].length; + } + return -1; + } + + protected _sliceInternal(self: Chunked<T>, begin: number, end: number) { + const slices: Vector<T>[] = []; + const { chunks, _chunkOffsets: chunkOffsets } = self; + for (let i = -1, n = chunks.length; ++i < n;) { + const chunk = chunks[i]; + const chunkLength = chunk.length; + const chunkOffset = chunkOffsets[i]; + // If the child is to the right of the slice boundary, we can stop + if (chunkOffset >= end) { break; } + // If the child is to the left of of the slice boundary, exclude + if (begin >= chunkOffset + chunkLength) { continue; } + // If the child is between both left and right boundaries, include w/o slicing + if (chunkOffset >= begin && (chunkOffset + chunkLength) <= end) { + slices.push(chunk); + continue; + } + // If the child overlaps one of the slice boundaries, include that slice + const from = Math.max(0, begin - chunkOffset); + const to = Math.min(end - chunkOffset, chunkLength); + slices.push(chunk.slice(from, to) as Vector<T>); + } + return self.clone(slices); + } +} + +/** @ignore */ +function calculateOffsets<T extends DataType>(vectors: Vector<T>[]) { + const offsets = new Uint32Array((vectors || []).length + 1); + let offset = offsets[0] = 0; + const length = offsets.length; + for (let index = 0; ++index < length;) { + offsets[index] = (offset += vectors[index - 1].length); + } + return offsets; +} + +/** @ignore */ +const typedSet = (src: TypedArray, dst: TypedArray, offset: number) => { + dst.set(src, offset); + return (offset + src.length); +}; + +/** @ignore */ +const arraySet = (src: any[], dst: any[], offset: number) => { + let idx = offset; + for (let i = -1, n = src.length; ++i < n;) { + dst[idx++] = src[i]; + } + return idx; +}; + +/** @ignore */ +interface TypedArray extends ArrayBufferView { + readonly length: number; + readonly [n: number]: number; + set(array: ArrayLike<number>, offset?: number): void; +} diff --git a/src/arrow/js/src/vector/date.ts b/src/arrow/js/src/vector/date.ts new file mode 100644 index 000000000..8c2b7a563 --- /dev/null +++ b/src/arrow/js/src/vector/date.ts @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { DateUnit } from '../enum'; +import { Chunked } from './chunked'; +import { BaseVector } from './base'; +import { VectorType as V } from '../interfaces'; +import { VectorBuilderOptions } from './index'; +import { vectorFromValuesWithType } from './index'; +import { VectorBuilderOptionsAsync } from './index'; +import { Date_, DateDay, DateMillisecond } from '../type'; + +/** @ignore */ +type FromArgs<T extends Date_> = [Iterable<Date>, T['unit']]; + +/** @ignore */ +export class DateVector<T extends Date_ = Date_> extends BaseVector<T> { + public static from<T extends DateUnit.DAY>(...args: FromArgs<DateDay>): V<DateDay>; + public static from<T extends DateUnit.MILLISECOND>(...args: FromArgs<DateMillisecond>): V<DateMillisecond>; + public static from<T extends Date_, TNull = any>(input: Iterable<Date | TNull>): V<T>; + public static from<T extends Date_, TNull = any>(input: AsyncIterable<Date | TNull>): Promise<V<T>>; + public static from<T extends Date_, TNull = any>(input: VectorBuilderOptions<T, Date | TNull>): Chunked<T>; + public static from<T extends Date_, TNull = any>(input: VectorBuilderOptionsAsync<T, Date | TNull>): Promise<Chunked<T>>; + /** @nocollapse */ + public static from<T extends Date_, TNull = any>(...args: FromArgs<T> | [Iterable<Date | TNull> | AsyncIterable<Date | TNull> | VectorBuilderOptions<T, Date | TNull> | VectorBuilderOptionsAsync<T, Date | TNull>]) { + if (args.length === 2) { + return vectorFromValuesWithType(() => args[1] === DateUnit.DAY ? new DateDay() : new DateMillisecond() as T, args[0]); + } + return vectorFromValuesWithType(() => new DateMillisecond() as T, args[0]); + } +} + +/** @ignore */ +export class DateDayVector extends DateVector<DateDay> {} + +/** @ignore */ +export class DateMillisecondVector extends DateVector<DateMillisecond> {} diff --git a/src/arrow/js/src/vector/decimal.ts b/src/arrow/js/src/vector/decimal.ts new file mode 100644 index 000000000..a1056fd4f --- /dev/null +++ b/src/arrow/js/src/vector/decimal.ts @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Decimal } from '../type'; +import { BaseVector } from './base'; + +/** @ignore */ +export class DecimalVector extends BaseVector<Decimal> {} diff --git a/src/arrow/js/src/vector/dictionary.ts b/src/arrow/js/src/vector/dictionary.ts new file mode 100644 index 000000000..4b39dbe97 --- /dev/null +++ b/src/arrow/js/src/vector/dictionary.ts @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import { Vector } from '../vector'; +import { BaseVector } from './base'; +import { VectorType as V } from '../interfaces'; +import { VectorBuilderOptions } from './index'; +import { vectorFromValuesWithType } from './index'; +import { VectorBuilderOptionsAsync } from './index'; +import { DataType, Dictionary, TKeys } from '../type'; + +/** @ignore */ +type FromArgs<T extends DataType = any, TKey extends TKeys = TKeys> = [Vector<T>, TKey, ArrayLike<number> | TKey['TArray']]; + +/** @ignore */ +export class DictionaryVector<T extends DataType = any, TKey extends TKeys = TKeys> extends BaseVector<Dictionary<T, TKey>> { + public static from<T extends DataType = any, TKey extends TKeys = TKeys>(...args: FromArgs<T, TKey>): V<Dictionary<T, TKey>>; + public static from<T extends DataType = any, TKey extends TKeys = TKeys>(input: VectorBuilderOptions<Dictionary<T, TKey>>): Vector<Dictionary<T, TKey>>; + public static from<T extends DataType = any, TKey extends TKeys = TKeys>(input: VectorBuilderOptionsAsync<Dictionary<T, TKey>>): Promise<Vector<Dictionary<T, TKey>>>; + /** @nocollapse */ + public static from<T extends DataType = any, TKey extends TKeys = TKeys>(...args: any[]) { + if (args.length === 3) { + const [values, indices, keys] = args as FromArgs<T, TKey>; + const type = new Dictionary(values.type, indices, null, null); + return Vector.new(Data.Dictionary(type, 0, keys.length, 0, null, keys, values)); + } + return vectorFromValuesWithType(() => args[0].type, args[0]); + } + + constructor(data: Data<Dictionary<T, TKey>>) { + super(data); + this.indices = Vector.new(data.clone(this.type.indices)); + } + + public readonly indices: V<TKey>; + + public get dictionary() { return <Vector<T>> this.data.dictionary; } + public reverseLookup(value: T) { return this.dictionary.indexOf(value); } + public getKey(idx: number): TKey['TValue'] | null { return this.indices.get(idx); } + public getValue(key: number): T['TValue'] | null { return this.dictionary.get(key); } + public setKey(idx: number, key: TKey['TValue'] | null) { return this.indices.set(idx, key); } + public setValue(key: number, value: T['TValue'] | null) { return this.dictionary.set(key, value); } +} + +(DictionaryVector.prototype as any).indices = null; diff --git a/src/arrow/js/src/vector/fixedsizebinary.ts b/src/arrow/js/src/vector/fixedsizebinary.ts new file mode 100644 index 000000000..779be19ff --- /dev/null +++ b/src/arrow/js/src/vector/fixedsizebinary.ts @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { BaseVector } from './base'; +import { FixedSizeBinary } from '../type'; + +/** @ignore */ +export class FixedSizeBinaryVector extends BaseVector<FixedSizeBinary> {} diff --git a/src/arrow/js/src/vector/fixedsizelist.ts b/src/arrow/js/src/vector/fixedsizelist.ts new file mode 100644 index 000000000..13637021f --- /dev/null +++ b/src/arrow/js/src/vector/fixedsizelist.ts @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { BaseVector } from './base'; +import { DataType, FixedSizeList } from '../type'; + +/** @ignore */ +export class FixedSizeListVector<T extends DataType = any> extends BaseVector<FixedSizeList<T>> {} diff --git a/src/arrow/js/src/vector/float.ts b/src/arrow/js/src/vector/float.ts new file mode 100644 index 000000000..8260d2b27 --- /dev/null +++ b/src/arrow/js/src/vector/float.ts @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import { Vector } from '../vector'; +import { Chunked } from './chunked'; +import { BaseVector } from './base'; +import { VectorBuilderOptions } from './index'; +import { vectorFromValuesWithType } from './index'; +import { VectorBuilderOptionsAsync } from './index'; +import { Float, Float16, Float32, Float64, FloatArray } from '../type'; +import { VectorType as V, TypedArrayConstructor } from '../interfaces'; + +/** @ignore */ +type FloatVectorConstructors = + typeof FloatVector | + typeof Float16Vector | + typeof Float32Vector | + typeof Float64Vector ; + +/** @ignore */ +type FromInput<T extends Float, TNull = any> = + FloatArray | + Iterable<T['TValue'] | TNull> | + AsyncIterable<T['TValue'] | TNull> | + VectorBuilderOptions<T, TNull> | + VectorBuilderOptionsAsync<T, TNull> ; + +/** @ignore */ +export type FloatArrayCtor = TypedArrayConstructor<FloatArray>; + +/** @ignore */ +export class FloatVector<T extends Float = Float> extends BaseVector<T> { + + // Guaranteed zero-copy variants + public static from(this: typeof FloatVector, input: Uint16Array): Float16Vector; + public static from(this: typeof FloatVector, input: Float32Array): Float32Vector; + public static from(this: typeof FloatVector, input: Float64Array): Float64Vector; + + // Zero-copy if input is a TypedArray of the same type as the + // Vector that from is called on, otherwise uses the Builders + public static from<TNull = any>(this: typeof Float16Vector, input: FromInput<Float16, TNull>): Float16Vector; + public static from<TNull = any>(this: typeof Float32Vector, input: FromInput<Float32, TNull>): Float32Vector; + public static from<TNull = any>(this: typeof Float64Vector, input: FromInput<Float64, TNull>): Float64Vector; + + // Not zero-copy + public static from<T extends Float, TNull = any>(this: typeof FloatVector, input: Iterable<T['TValue'] | TNull>): V<T>; + public static from<T extends Float, TNull = any>(this: typeof FloatVector, input: AsyncIterable<T['TValue'] | TNull>): Promise<V<T>>; + public static from<T extends Float, TNull = any>(this: typeof FloatVector, input: VectorBuilderOptions<T, TNull>): Chunked<T>; + public static from<T extends Float, TNull = any>(this: typeof FloatVector, input: VectorBuilderOptionsAsync<T, TNull>): Promise<Chunked<T>>; + /** @nocollapse */ + public static from<T extends Float, TNull = any>(this: FloatVectorConstructors, input: FromInput<T, TNull>) { + + let ArrowType = vectorTypeToDataType(this); + + if ((input instanceof ArrayBuffer) || ArrayBuffer.isView(input)) { + const InputType = arrayTypeToDataType(input.constructor as FloatArrayCtor) || ArrowType; + // Special case, infer the Arrow DataType from the input if calling the base + // FloatVector.from with a TypedArray, e.g. `FloatVector.from(new Float32Array())` + if (ArrowType === null) { + ArrowType = InputType; + } + // If the DataType inferred from the Vector constructor matches the + // DataType inferred from the input arguments, return zero-copy view + if (ArrowType && ArrowType === InputType) { + const type = new ArrowType(); + const length = input.byteLength / type.ArrayType.BYTES_PER_ELEMENT; + // If the ArrowType is Float16 but the input type isn't a Uint16Array, + // let the Float16Builder handle casting the input values to Uint16s. + if (!convertTo16Bit(ArrowType, input.constructor)) { + return Vector.new(Data.Float(type, 0, length, 0, null, input as FloatArray)); + } + } + } + + if (ArrowType) { + // If the DataType inferred from the Vector constructor is different than + // the DataType inferred from the input TypedArray, or if input isn't a + // TypedArray, use the Builders to construct the result Vector + return vectorFromValuesWithType(() => new ArrowType!() as T, input); + } + + if ((input instanceof DataView) || (input instanceof ArrayBuffer)) { + throw new TypeError(`Cannot infer float type from instance of ${input.constructor.name}`); + } + + throw new TypeError('Unrecognized FloatVector input'); + } +} + +/** @ignore */ +export class Float16Vector extends FloatVector<Float16> { + // Since JS doesn't have half floats, `toArray()` returns a zero-copy slice + // of the underlying Uint16Array data. This behavior ensures we don't incur + // extra compute or copies if you're calling `toArray()` in order to create + // a buffer for something like WebGL. Buf if you're using JS and want typed + // arrays of 4-to-8-byte precision, these methods will enumerate the values + // and clamp to the desired byte lengths. + public toFloat32Array() { return new Float32Array(this as Iterable<number>); } + public toFloat64Array() { return new Float64Array(this as Iterable<number>); } +} + +/** @ignore */ +export class Float32Vector extends FloatVector<Float32> {} +/** @ignore */ +export class Float64Vector extends FloatVector<Float64> {} + +const convertTo16Bit = (typeCtor: any, dataCtor: any) => { + return (typeCtor === Float16) && (dataCtor !== Uint16Array); +}; + +/** @ignore */ +const arrayTypeToDataType = (ctor: FloatArrayCtor) => { + switch (ctor) { + case Uint16Array: return Float16; + case Float32Array: return Float32; + case Float64Array: return Float64; + default: return null; + } +}; + +/** @ignore */ +const vectorTypeToDataType = (ctor: FloatVectorConstructors) => { + switch (ctor) { + case Float16Vector: return Float16; + case Float32Vector: return Float32; + case Float64Vector: return Float64; + default: return null; + } +}; diff --git a/src/arrow/js/src/vector/index.ts b/src/arrow/js/src/vector/index.ts new file mode 100644 index 000000000..30f5e3cfa --- /dev/null +++ b/src/arrow/js/src/vector/index.ts @@ -0,0 +1,207 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +export { Vector } from '../vector'; +export { BaseVector } from './base'; +export { BinaryVector } from './binary'; +export { BoolVector } from './bool'; +export { Chunked } from './chunked'; +export { DateVector, DateDayVector, DateMillisecondVector } from './date'; +export { DecimalVector } from './decimal'; +export { DictionaryVector } from './dictionary'; +export { FixedSizeBinaryVector } from './fixedsizebinary'; +export { FixedSizeListVector } from './fixedsizelist'; +export { FloatVector, Float16Vector, Float32Vector, Float64Vector } from './float'; +export { IntervalVector, IntervalDayTimeVector, IntervalYearMonthVector } from './interval'; +export { IntVector, Int8Vector, Int16Vector, Int32Vector, Int64Vector, Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector } from './int'; +export { ListVector } from './list'; +export { MapVector } from './map'; +export { NullVector } from './null'; +export { StructVector } from './struct'; +export { TimestampVector, TimestampSecondVector, TimestampMillisecondVector, TimestampMicrosecondVector, TimestampNanosecondVector } from './timestamp'; +export { TimeVector, TimeSecondVector, TimeMillisecondVector, TimeMicrosecondVector, TimeNanosecondVector } from './time'; +export { UnionVector, DenseUnionVector, SparseUnionVector } from './union'; +export { Utf8Vector } from './utf8'; +export { MapRow, StructRow } from './row'; + +import * as fn from '../util/fn'; +import { Data } from '../data'; +import { Type } from '../enum'; +import { Vector } from '../vector'; +import { DataType } from '../type'; +import { Chunked } from './chunked'; +import { BaseVector } from './base'; +import { setBool } from '../util/bit'; +import { isIterable, isAsyncIterable } from '../util/compat'; +import { Builder, IterableBuilderOptions } from '../builder'; +import { VectorType as V, VectorCtorArgs } from '../interfaces'; +import { instance as getVisitor } from '../visitor/get'; +import { instance as setVisitor } from '../visitor/set'; +import { instance as indexOfVisitor } from '../visitor/indexof'; +import { instance as toArrayVisitor } from '../visitor/toarray'; +import { instance as iteratorVisitor } from '../visitor/iterator'; +import { instance as byteWidthVisitor } from '../visitor/bytewidth'; +import { instance as getVectorConstructor } from '../visitor/vectorctor'; + +declare module '../vector' { + namespace Vector { + export { newVector as new }; + export { vectorFrom as from }; + } +} + +declare module './base' { + namespace BaseVector { + export { vectorFrom as from }; + } + interface BaseVector<T extends DataType> { + get(index: number): T['TValue'] | null; + set(index: number, value: T['TValue'] | null): void; + indexOf(value: T['TValue'] | null, fromIndex?: number): number; + toArray(): T['TArray']; + getByteWidth(): number; + [Symbol.iterator](): IterableIterator<T['TValue'] | null>; + } +} + +/** @nocollapse */ +Vector.new = newVector; + +/** @nocollapse */ +Vector.from = vectorFrom; + +/** @ignore */ +function newVector<T extends DataType>(data: Data<T>, ...args: VectorCtorArgs<V<T>>): V<T> { + return new (getVectorConstructor.getVisitFn<T>(data)())(data, ...args) as V<T>; +} + +/** @ignore */ +export interface VectorBuilderOptions<T extends DataType, TNull = any> extends IterableBuilderOptions<T, TNull> { values: Iterable<T['TValue'] | TNull> } +/** @ignore */ +export interface VectorBuilderOptionsAsync<T extends DataType, TNull = any> extends IterableBuilderOptions<T, TNull> { values: AsyncIterable<T['TValue'] | TNull> } + +/** @ignore */ +export function vectorFromValuesWithType<T extends DataType, TNull = any>(newDataType: () => T, input: Iterable<T['TValue'] | TNull> | AsyncIterable<T['TValue'] | TNull> | VectorBuilderOptions<T, TNull> | VectorBuilderOptionsAsync<T, TNull>) { + if (isIterable(input)) { + return Vector.from({ 'nullValues': [null, undefined], type: newDataType(), 'values': input }) as V<T>; + } else if (isAsyncIterable(input)) { + return Vector.from({ 'nullValues': [null, undefined], type: newDataType(), 'values': input }) as Promise<V<T>>; + } + const { + 'values': values = [], + 'type': type = newDataType(), + 'nullValues': nullValues = [null, undefined], + } = { ...input }; + return isIterable(values) + ? Vector.from({ nullValues, ...input, type } as VectorBuilderOptions<T, TNull>) + : Vector.from({ nullValues, ...input, type } as VectorBuilderOptionsAsync<T, TNull>); +} + +/** @ignore */ +function vectorFrom<T extends DataType = any, TNull = any>(input: VectorBuilderOptions<T, TNull>): Vector<T>; +function vectorFrom<T extends DataType = any, TNull = any>(input: VectorBuilderOptionsAsync<T, TNull>): Promise<Vector<T>>; +function vectorFrom<T extends DataType = any, TNull = any>(input: VectorBuilderOptions<T, TNull> | VectorBuilderOptionsAsync<T, TNull>) { + const { 'values': values = [], ...options } = { 'nullValues': [null, undefined], ...input } as VectorBuilderOptions<T, TNull> | VectorBuilderOptionsAsync<T, TNull>; + if (isIterable<T['TValue'] | TNull>(values)) { + const chunks = [...Builder.throughIterable(options)(values)]; + return (chunks.length === 1 ? chunks[0] : Chunked.concat<T>(chunks)) as Vector<T>; + } + return (async (chunks: V<T>[]) => { + const transform = Builder.throughAsyncIterable(options); + for await (const chunk of transform(values)) { + chunks.push(chunk); + } + return (chunks.length === 1 ? chunks[0] : Chunked.concat<T>(chunks)) as Vector<T>; + })([]); +} + +// +// We provide the following method implementations for code navigability purposes only. +// They're overridden at runtime below with the specific Visitor implementation for each type, +// short-circuiting the usual Visitor traversal and reducing intermediate lookups and calls. +// This comment is here to remind you to not set breakpoints in these function bodies, or to inform +// you why the breakpoints you have already set are not being triggered. Have a great day! +// + +BaseVector.prototype.get = function baseVectorGet<T extends DataType>(this: BaseVector<T>, index: number): T['TValue'] | null { + return getVisitor.visit(this, index); +}; + +BaseVector.prototype.set = function baseVectorSet<T extends DataType>(this: BaseVector<T>, index: number, value: T['TValue'] | null): void { + return setVisitor.visit(this, index, value); +}; + +BaseVector.prototype.indexOf = function baseVectorIndexOf<T extends DataType>(this: BaseVector<T>, value: T['TValue'] | null, fromIndex?: number): number { + return indexOfVisitor.visit(this, value, fromIndex); +}; + +BaseVector.prototype.toArray = function baseVectorToArray<T extends DataType>(this: BaseVector<T>): T['TArray'] { + return toArrayVisitor.visit(this); +}; + +BaseVector.prototype.getByteWidth = function baseVectorGetByteWidth<T extends DataType>(this: BaseVector<T>): number { + return byteWidthVisitor.visit(this.type); +}; + +BaseVector.prototype[Symbol.iterator] = function baseVectorSymbolIterator<T extends DataType>(this: BaseVector<T>): IterableIterator<T['TValue'] | null> { + return iteratorVisitor.visit(this); +}; + +(BaseVector.prototype as any)._bindDataAccessors = bindBaseVectorDataAccessors; + +// Perf: bind and assign the operator Visitor methods to each of the Vector subclasses for each Type +(Object.keys(Type) as any[]) + .map((T: any) => Type[T] as any) + .filter((T: any): T is Type => typeof T === 'number') + .filter((typeId) => typeId !== Type.NONE) + .forEach((typeId) => { + const VectorCtor = getVectorConstructor.visit(typeId); + VectorCtor.prototype['get'] = fn.partial1(getVisitor.getVisitFn(typeId)); + VectorCtor.prototype['set'] = fn.partial2(setVisitor.getVisitFn(typeId)); + VectorCtor.prototype['indexOf'] = fn.partial2(indexOfVisitor.getVisitFn(typeId)); + VectorCtor.prototype['toArray'] = fn.partial0(toArrayVisitor.getVisitFn(typeId)); + VectorCtor.prototype['getByteWidth'] = partialType0(byteWidthVisitor.getVisitFn(typeId)); + VectorCtor.prototype[Symbol.iterator] = fn.partial0(iteratorVisitor.getVisitFn(typeId)); + }); + +/** @ignore */ +function partialType0<T extends Vector>(visit: (node: T['type']) => any) { + return function(this: T) { return visit(this.type); }; +} + +/** @ignore */ +function wrapNullableGet<T extends DataType, V extends Vector<T>, F extends (i: number) => any>(fn: F): (...args: Parameters<F>) => ReturnType<F> { + return function(this: V, i: number) { return this.isValid(i) ? fn.call(this, i) : null; }; +} + +/** @ignore */ +function wrapNullableSet<T extends DataType, V extends BaseVector<T>, F extends (i: number, a: any) => void>(fn: F): (...args: Parameters<F>) => void { + return function(this: V, i: number, a: any) { + if (setBool(this.nullBitmap, this.offset + i, !((a == null)))) { + fn.call(this, i, a); + } + }; +} + +/** @ignore */ +function bindBaseVectorDataAccessors<T extends DataType>(this: BaseVector<T>) { + const nullBitmap = this.nullBitmap; + if (nullBitmap && nullBitmap.byteLength > 0) { + this.get = wrapNullableGet(this.get); + this.set = wrapNullableSet(this.set); + } +} diff --git a/src/arrow/js/src/vector/int.ts b/src/arrow/js/src/vector/int.ts new file mode 100644 index 000000000..dbfba58c9 --- /dev/null +++ b/src/arrow/js/src/vector/int.ts @@ -0,0 +1,195 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import { Vector } from '../vector'; +import { Chunked } from './chunked'; +import { BaseVector } from './base'; +import { VectorBuilderOptions } from './index'; +import { vectorFromValuesWithType } from './index'; +import { VectorBuilderOptionsAsync } from './index'; +import { BigInt64Array, BigUint64Array } from '../util/compat'; +import { toBigInt64Array, toBigUint64Array } from '../util/buffer'; +import { Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, IntArray } from '../type'; +import { VectorType as V, TypedArrayConstructor, BigIntArrayConstructor, BigIntArray } from '../interfaces'; + +/** @ignore */ +type IntVectorConstructors = + typeof IntVector | + typeof Int8Vector | + typeof Int16Vector | + typeof Int32Vector | + typeof Uint8Vector | + typeof Uint16Vector | + typeof Uint32Vector | + typeof Int64Vector | + typeof Uint64Vector ; + +/** @ignore */ +type FromInput<T extends Int, TNull = any> = + IntArray | BigIntArray | + Iterable<T['TValue'] | TNull> | + AsyncIterable<T['TValue'] | TNull> | + VectorBuilderOptions<T, TNull> | + VectorBuilderOptionsAsync<T, TNull> ; + +/** @ignore */ +type FromArgs<T extends Int, TNull = any> = [FromInput<T, TNull>, boolean?]; + +/** @ignore */ +export type IntArrayCtor = TypedArrayConstructor<IntArray> | BigIntArrayConstructor<BigIntArray>; + +/** @ignore */ +export class IntVector<T extends Int = Int> extends BaseVector<T> { + + // Guaranteed zero-copy variants + public static from(this: typeof IntVector, input: Int8Array): Int8Vector; + public static from(this: typeof IntVector, input: Int16Array): Int16Vector; + public static from(this: typeof IntVector, input: Int32Array): Int32Vector; + public static from(this: typeof IntVector, input: BigInt64Array): Int64Vector; + public static from(this: typeof IntVector, input: Int32Array, is64bit: true): Int64Vector; + public static from(this: typeof IntVector, input: Uint8Array): Uint8Vector; + public static from(this: typeof IntVector, input: Uint16Array): Uint16Vector; + public static from(this: typeof IntVector, input: Uint32Array): Uint32Vector; + public static from(this: typeof IntVector, input: BigUint64Array): Uint64Vector; + public static from(this: typeof IntVector, input: Uint32Array, is64bit: true): Uint64Vector; + + // Zero-copy if input is a TypedArray of the same type as the + // Vector that from is called on, otherwise uses the Builders + public static from<TNull = any>(this: typeof Int8Vector, input: FromInput<Int8, TNull>): Int8Vector; + public static from<TNull = any>(this: typeof Int16Vector, input: FromInput<Int16, TNull>): Int16Vector; + public static from<TNull = any>(this: typeof Int32Vector, input: FromInput<Int32, TNull>): Int32Vector; + public static from<TNull = any>(this: typeof Int64Vector, input: FromInput<Int64, TNull>): Int64Vector; + public static from<TNull = any>(this: typeof Uint8Vector, input: FromInput<Uint8, TNull>): Uint8Vector; + public static from<TNull = any>(this: typeof Uint16Vector, input: FromInput<Uint16, TNull>): Uint16Vector; + public static from<TNull = any>(this: typeof Uint32Vector, input: FromInput<Uint32, TNull>): Uint32Vector; + public static from<TNull = any>(this: typeof Uint64Vector, input: FromInput<Uint64, TNull>): Uint64Vector; + + // Not zero-copy + public static from<T extends Int, TNull = any>(this: typeof IntVector, input: Iterable<T['TValue'] | TNull>): V<T>; + public static from<T extends Int, TNull = any>(this: typeof IntVector, input: AsyncIterable<T['TValue'] | TNull>): Promise<V<T>>; + public static from<T extends Int, TNull = any>(this: typeof IntVector, input: VectorBuilderOptions<T, TNull>): Chunked<T>; + public static from<T extends Int, TNull = any>(this: typeof IntVector, input: VectorBuilderOptionsAsync<T, TNull>): Promise<Chunked<T>>; + /** @nocollapse */ + public static from<T extends Int, TNull = any>(this: IntVectorConstructors, ...args: FromArgs<T, TNull>) { + + const [input, is64bit = false] = args; + let ArrowType = vectorTypeToDataType(this, is64bit); + + if ((input instanceof ArrayBuffer) || ArrayBuffer.isView(input)) { + const InputType = arrayTypeToDataType(input.constructor as IntArrayCtor, is64bit) || ArrowType; + // Special case, infer the Arrow DataType from the input if calling the base + // IntVector.from with a TypedArray, e.g. `IntVector.from(new Int32Array())` + if (ArrowType === null) { + ArrowType = InputType; + } + // If the DataType inferred from the Vector constructor matches the + // DataType inferred from the input arguments, return zero-copy view + if (ArrowType && ArrowType === InputType) { + const type = new ArrowType(); + let length = input.byteLength / type.ArrayType.BYTES_PER_ELEMENT; + // If the ArrowType is 64bit but the input type is 32bit pairs, update the logical length + if (convert32To64Bit(ArrowType, input.constructor)) { + length *= 0.5; + } + return Vector.new(Data.Int(type, 0, length, 0, null, input as IntArray)); + } + } + + if (ArrowType) { + // If the DataType inferred from the Vector constructor is different than + // the DataType inferred from the input TypedArray, or if input isn't a + // TypedArray, use the Builders to construct the result Vector + return vectorFromValuesWithType(() => new ArrowType!() as T, input); + } + + if ((input instanceof DataView) || (input instanceof ArrayBuffer)) { + throw new TypeError(`Cannot infer integer type from instance of ${input.constructor.name}`); + } + + throw new TypeError('Unrecognized IntVector input'); + } +} + +/** @ignore */ +export class Int8Vector extends IntVector<Int8> {} +/** @ignore */ +export class Int16Vector extends IntVector<Int16> {} +/** @ignore */ +export class Int32Vector extends IntVector<Int32> {} +/** @ignore */ +export class Int64Vector extends IntVector<Int64> { + public toBigInt64Array() { + return toBigInt64Array(this.values); + } + private _values64!: BigInt64Array; + public get values64(): BigInt64Array { + return this._values64 || (this._values64 = this.toBigInt64Array()); + } +} + +/** @ignore */ +export class Uint8Vector extends IntVector<Uint8> {} +/** @ignore */ +export class Uint16Vector extends IntVector<Uint16> {} +/** @ignore */ +export class Uint32Vector extends IntVector<Uint32> {} +/** @ignore */ +export class Uint64Vector extends IntVector<Uint64> { + public toBigUint64Array() { + return toBigUint64Array(this.values); + } + private _values64!: BigUint64Array; + public get values64(): BigUint64Array { + return this._values64 || (this._values64 = this.toBigUint64Array()); + } +} + +const convert32To64Bit = (typeCtor: any, dataCtor: any) => { + return (typeCtor === Int64 || typeCtor === Uint64) && + (dataCtor === Int32Array || dataCtor === Uint32Array); +}; + +/** @ignore */ +const arrayTypeToDataType = (ctor: IntArrayCtor, is64bit: boolean) => { + switch (ctor) { + case Int8Array: return Int8; + case Int16Array: return Int16; + case Int32Array: return is64bit ? Int64 : Int32; + case BigInt64Array: return Int64; + case Uint8Array: return Uint8; + case Uint16Array: return Uint16; + case Uint32Array: return is64bit ? Uint64 : Uint32; + case BigUint64Array: return Uint64; + default: return null; + } +}; + +/** @ignore */ +const vectorTypeToDataType = (ctor: IntVectorConstructors, is64bit: boolean) => { + switch (ctor) { + case Int8Vector: return Int8; + case Int16Vector: return Int16; + case Int32Vector: return is64bit ? Int64 : Int32; + case Int64Vector: return Int64; + case Uint8Vector: return Uint8; + case Uint16Vector: return Uint16; + case Uint32Vector: return is64bit ? Uint64 : Uint32; + case Uint64Vector: return Uint64; + default: return null; + } +}; diff --git a/src/arrow/js/src/vector/interval.ts b/src/arrow/js/src/vector/interval.ts new file mode 100644 index 000000000..70384ab97 --- /dev/null +++ b/src/arrow/js/src/vector/interval.ts @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { BaseVector } from './base'; +import { Interval, IntervalDayTime, IntervalYearMonth } from '../type'; + +/** @ignore */ +export class IntervalVector<T extends Interval = Interval> extends BaseVector<T> {} +/** @ignore */ +export class IntervalDayTimeVector extends IntervalVector<IntervalDayTime> {} +/** @ignore */ +export class IntervalYearMonthVector extends IntervalVector<IntervalYearMonth> {} diff --git a/src/arrow/js/src/vector/list.ts b/src/arrow/js/src/vector/list.ts new file mode 100644 index 000000000..6ea189044 --- /dev/null +++ b/src/arrow/js/src/vector/list.ts @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { BaseVector } from './base'; +import { DataType, List } from '../type'; + +/** @ignore */ +export class ListVector<T extends DataType = any> extends BaseVector<List<T>> {} diff --git a/src/arrow/js/src/vector/map.ts b/src/arrow/js/src/vector/map.ts new file mode 100644 index 000000000..9975919f7 --- /dev/null +++ b/src/arrow/js/src/vector/map.ts @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { MapRow } from './row'; +import { Field } from '../schema'; +import { Vector } from '../vector'; +import { BaseVector } from './base'; +import { DataType, Map_, Struct, List } from '../type'; + +/** @ignore */ +export class MapVector<K extends DataType = any, V extends DataType = any> extends BaseVector<Map_<K, V>> { + public asList() { + const child = this.type.children[0] as Field<Struct<{ key: K; value: V }>>; + return Vector.new(this.data.clone(new List<Struct<{ key: K; value: V }>>(child))); + } + public bind(index: number): Map_<K, V>['TValue'] { + const child = this.getChildAt<Struct<{ key: K; value: V }>>(0)!; + const { [index]: begin, [index + 1]: end } = this.valueOffsets; + return new MapRow(child.slice(begin, end)); + } +} diff --git a/src/arrow/js/src/vector/null.ts b/src/arrow/js/src/vector/null.ts new file mode 100644 index 000000000..ffa3d0576 --- /dev/null +++ b/src/arrow/js/src/vector/null.ts @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Null } from '../type'; +import { BaseVector } from './base'; + +/** @ignore */ +export class NullVector extends BaseVector<Null> {} diff --git a/src/arrow/js/src/vector/row.ts b/src/arrow/js/src/vector/row.ts new file mode 100644 index 000000000..23d1b5440 --- /dev/null +++ b/src/arrow/js/src/vector/row.ts @@ -0,0 +1,296 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from '../vector'; +import { StructVector } from './struct'; +import { valueToString } from '../util/pretty'; +import { DataType, Struct, RowLike } from '../type'; + +/** @ignore */ const kParent = Symbol.for('parent'); +/** @ignore */ const kRowIndex = Symbol.for('rowIndex'); +/** @ignore */ const kKeyToIdx = Symbol.for('keyToIdx'); +/** @ignore */ const kIdxToVal = Symbol.for('idxToVal'); +/** @ignore */ const kCustomInspect = Symbol.for('nodejs.util.inspect.custom'); + +abstract class Row<K extends PropertyKey = any, V = any> implements Map<K, V> { + + public readonly size: number; + public readonly [Symbol.toStringTag]: string; + + protected [kRowIndex]: number; + protected [kParent]: Vector<Struct>; + protected [kKeyToIdx]: Map<K, number>; + protected [kIdxToVal]: V[]; + + constructor(parent: Vector<Struct>, numKeys: number) { + this[kParent] = parent; + this.size = numKeys; + } + + public abstract keys(): IterableIterator<K>; + public abstract values(): IterableIterator<V>; + public abstract getKey(idx: number): K; + public abstract getIndex(key: K): number; + public abstract getValue(idx: number): V; + public abstract setValue(idx: number, val: V): void; + + public entries() { return this[Symbol.iterator](); } + + public has(key: K) { return this.get(key) !== undefined; } + + public get(key: K) { + let val = undefined; + if (key != null) { + const ktoi = this[kKeyToIdx] || (this[kKeyToIdx] = new Map()); + let idx = ktoi.get(key); + if (idx !== undefined) { + const itov = this[kIdxToVal] || (this[kIdxToVal] = new Array(this.size)); + ((val = itov[idx]) !== undefined) || (itov[idx] = val = this.getValue(idx)); + } else if ((idx = this.getIndex(key)) > -1) { + ktoi.set(key, idx); + const itov = this[kIdxToVal] || (this[kIdxToVal] = new Array(this.size)); + ((val = itov[idx]) !== undefined) || (itov[idx] = val = this.getValue(idx)); + } + } + return val; + } + + public set(key: K, val: V) { + if (key != null) { + const ktoi = this[kKeyToIdx] || (this[kKeyToIdx] = new Map()); + let idx = ktoi.get(key); + if (idx === undefined) { + ktoi.set(key, idx = this.getIndex(key)); + } + if (idx > -1) { + const itov = this[kIdxToVal] || (this[kIdxToVal] = new Array(this.size)); + itov[idx] = <any> this.setValue(idx, val); + } + } + return this; + } + + public clear(): void { throw new Error(`Clearing ${this[Symbol.toStringTag]} not supported.`); } + + public delete(_: K): boolean { throw new Error(`Deleting ${this[Symbol.toStringTag]} values not supported.`); } + + public *[Symbol.iterator](): IterableIterator<[K, V]> { + + const ki = this.keys(); + const vi = this.values(); + const ktoi = this[kKeyToIdx] || (this[kKeyToIdx] = new Map()); + const itov = this[kIdxToVal] || (this[kIdxToVal] = new Array(this.size)); + + for (let k: K, v: V, i = 0, kr: IteratorResult<K>, vr: IteratorResult<V>; + !((kr = ki.next()).done || (vr = vi.next()).done); + ++i + ) { + k = kr.value; + v = vr.value; + itov[i] = v; + ktoi.has(k) || ktoi.set(k, i); + yield [k, v]; + } + } + + public forEach(callbackfn: (value: V, key: K, map: Map<K, V>) => void, thisArg?: any): void { + + const ki = this.keys(); + const vi = this.values(); + const callback = thisArg === undefined ? callbackfn : + (v: V, k: K, m: Map<K, V>) => callbackfn.call(thisArg, v, k, m); + const ktoi = this[kKeyToIdx] || (this[kKeyToIdx] = new Map()); + const itov = this[kIdxToVal] || (this[kIdxToVal] = new Array(this.size)); + + for (let k: K, v: V, i = 0, kr: IteratorResult<K>, vr: IteratorResult<V>; + !((kr = ki.next()).done || (vr = vi.next()).done); + ++i + ) { + k = kr.value; + v = vr.value; + itov[i] = v; + ktoi.has(k) || ktoi.set(k, i); + callback(v, k, this); + } + } + + public toArray() { return [...this.values()]; } + public toJSON() { + const obj = {} as any; + this.forEach((val, key) => obj[key] = val); + return obj; + } + + public inspect() { return this.toString(); } + public [kCustomInspect]() { return this.toString(); } + public toString() { + const str: string[] = []; + this.forEach((val, key) => { + key = valueToString(key); + val = valueToString(val); + str.push(`${key}: ${val}`); + }); + return `{ ${str.join(', ')} }`; + } + + protected static [Symbol.toStringTag] = ((proto: Row) => { + Object.defineProperties(proto, { + 'size': { writable: true, enumerable: false, configurable: false, value: 0 }, + [kParent]: { writable: true, enumerable: false, configurable: false, value: null }, + [kRowIndex]: { writable: true, enumerable: false, configurable: false, value: -1 }, + }); + return (proto as any)[Symbol.toStringTag] = 'Row'; + })(Row.prototype); +} + +export class MapRow<K extends DataType = any, V extends DataType = any> extends Row<K['TValue'], V['TValue'] | null> { + constructor(slice: Vector<Struct<{ key: K; value: V }>>) { + super(slice, slice.length); + return createRowProxy(this); + } + public keys() { + return this[kParent].getChildAt(0)![Symbol.iterator](); + } + public values() { + return this[kParent].getChildAt(1)![Symbol.iterator](); + } + public getKey(idx: number): K['TValue'] { + return this[kParent].getChildAt(0)!.get(idx); + } + public getIndex(key: K['TValue']): number { + return this[kParent].getChildAt(0)!.indexOf(key); + } + public getValue(index: number): V['TValue'] | null { + return this[kParent].getChildAt(1)!.get(index); + } + public setValue(index: number, value: V['TValue'] | null): void { + this[kParent].getChildAt(1)!.set(index, value); + } +} + +export class StructRow<T extends { [key: string]: DataType } = any> extends Row<keyof T, T[keyof T]['TValue'] | null> { + constructor(parent: StructVector<T>) { + super(parent, parent.type.children.length); + return defineRowProxyProperties(this); + } + public *keys() { + for (const field of this[kParent].type.children) { + yield field.name as keyof T; + } + } + public *values() { + for (const field of this[kParent].type.children) { + yield (this as RowLike<T>)[field.name]; + } + } + public getKey(idx: number): keyof T { + return this[kParent].type.children[idx].name as keyof T; + } + public getIndex(key: keyof T): number { + return this[kParent].type.children.findIndex((f) => f.name === key); + } + public getValue(index: number): T[keyof T]['TValue'] | null { + return this[kParent].getChildAt(index)!.get(this[kRowIndex]); + } + public setValue(index: number, value: T[keyof T]['TValue'] | null): void { + return this[kParent].getChildAt(index)!.set(this[kRowIndex], value); + } +} + +Object.setPrototypeOf(Row.prototype, Map.prototype); + +/** @ignore */ +const defineRowProxyProperties = (() => { + const desc = { enumerable: true, configurable: false, get: null as any, set: null as any }; + return <T extends Row>(row: T) => { + let idx = -1; + const ktoi = row[kKeyToIdx] || (row[kKeyToIdx] = new Map()); + const getter = (key: any) => function(this: T) { return this.get(key); }; + const setter = (key: any) => function(this: T, val: any) { return this.set(key, val); }; + for (const key of row.keys()) { + ktoi.set(key, ++idx); + desc.get = getter(key); + desc.set = setter(key); + Object.prototype.hasOwnProperty.call(row, key) || (desc.enumerable = true, Object.defineProperty(row, key, desc)); + Object.prototype.hasOwnProperty.call(row, idx) || (desc.enumerable = false, Object.defineProperty(row, idx, desc)); + } + desc.get = desc.set = null; + return row; + }; +})(); + +/** @ignore */ +const createRowProxy = (() => { + if (typeof Proxy === 'undefined') { + return defineRowProxyProperties; + } + const has = Row.prototype.has; + const get = Row.prototype.get; + const set = Row.prototype.set; + const getKey = Row.prototype.getKey; + const RowProxyHandler: ProxyHandler<Row> = { + isExtensible() { return false; }, + deleteProperty() { return false; }, + preventExtensions() { return true; }, + ownKeys(row: Row) { return [...row.keys()].map((x) => `${x}`); }, + has(row: Row, key: PropertyKey) { + switch (key) { + case 'getKey': case 'getIndex': case 'getValue': case 'setValue': case 'toArray': case 'toJSON': case 'inspect': + case 'constructor': case 'isPrototypeOf': case 'propertyIsEnumerable': case 'toString': case 'toLocaleString': case 'valueOf': + case 'size': case 'has': case 'get': case 'set': case 'clear': case 'delete': case 'keys': case 'values': case 'entries': case 'forEach': + case '__proto__': case '__defineGetter__': case '__defineSetter__': case 'hasOwnProperty': case '__lookupGetter__': case '__lookupSetter__': + case Symbol.iterator: case Symbol.toStringTag: case kParent: case kRowIndex: case kIdxToVal: case kKeyToIdx: case kCustomInspect: + return true; + } + if (typeof key === 'number' && !row.has(key)) { + key = row.getKey(key); + } + return row.has(key); + }, + get(row: Row, key: PropertyKey, receiver: any) { + switch (key) { + case 'getKey': case 'getIndex': case 'getValue': case 'setValue': case 'toArray': case 'toJSON': case 'inspect': + case 'constructor': case 'isPrototypeOf': case 'propertyIsEnumerable': case 'toString': case 'toLocaleString': case 'valueOf': + case 'size': case 'has': case 'get': case 'set': case 'clear': case 'delete': case 'keys': case 'values': case 'entries': case 'forEach': + case '__proto__': case '__defineGetter__': case '__defineSetter__': case 'hasOwnProperty': case '__lookupGetter__': case '__lookupSetter__': + case Symbol.iterator: case Symbol.toStringTag: case kParent: case kRowIndex: case kIdxToVal: case kKeyToIdx: case kCustomInspect: + return Reflect.get(row, key, receiver); + } + if (typeof key === 'number' && !has.call(receiver, key)) { + key = getKey.call(receiver, key); + } + return get.call(receiver, key); + }, + set(row: Row, key: PropertyKey, val: any, receiver: any) { + switch (key) { + case kParent: case kRowIndex: case kIdxToVal: case kKeyToIdx: + return Reflect.set(row, key, val, receiver); + case 'getKey': case 'getIndex': case 'getValue': case 'setValue': case 'toArray': case 'toJSON': case 'inspect': + case 'constructor': case 'isPrototypeOf': case 'propertyIsEnumerable': case 'toString': case 'toLocaleString': case 'valueOf': + case 'size': case 'has': case 'get': case 'set': case 'clear': case 'delete': case 'keys': case 'values': case 'entries': case 'forEach': + case '__proto__': case '__defineGetter__': case '__defineSetter__': case 'hasOwnProperty': case '__lookupGetter__': case '__lookupSetter__': + case Symbol.iterator: case Symbol.toStringTag: + return false; + } + if (typeof key === 'number' && !has.call(receiver, key)) { + key = getKey.call(receiver, key); + } + return has.call(receiver, key) ? !!set.call(receiver, key, val) : false; + }, + }; + return <T extends Row>(row: T) => new Proxy(row, RowProxyHandler) as T; +})(); diff --git a/src/arrow/js/src/vector/struct.ts b/src/arrow/js/src/vector/struct.ts new file mode 100644 index 000000000..b825f092e --- /dev/null +++ b/src/arrow/js/src/vector/struct.ts @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { StructRow } from './row'; +import { BaseVector } from './base'; +import { DataType, Struct } from '../type'; + +/** @ignore */ const kRowIndex = Symbol.for('rowIndex'); +/** @ignore */ +export class StructVector<T extends { [key: string]: DataType } = any> extends BaseVector<Struct<T>> { + private _row!: StructRow<T>; + public bind(index: number): Struct<T>['TValue'] { + const proto = this._row || (this._row = new StructRow<T>(this)); + const bound = Object.create(proto); + bound[kRowIndex] = index; + return bound; + } +} diff --git a/src/arrow/js/src/vector/time.ts b/src/arrow/js/src/vector/time.ts new file mode 100644 index 000000000..0abded940 --- /dev/null +++ b/src/arrow/js/src/vector/time.ts @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { BaseVector } from './base'; +import { Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond } from '../type'; + +/** @ignore */ +export class TimeVector<T extends Time = Time> extends BaseVector<T> {} +/** @ignore */ +export class TimeSecondVector extends TimeVector<TimeSecond> {} +/** @ignore */ +export class TimeMillisecondVector extends TimeVector<TimeMillisecond> {} +/** @ignore */ +export class TimeMicrosecondVector extends TimeVector<TimeMicrosecond> {} +/** @ignore */ +export class TimeNanosecondVector extends TimeVector<TimeNanosecond> {} diff --git a/src/arrow/js/src/vector/timestamp.ts b/src/arrow/js/src/vector/timestamp.ts new file mode 100644 index 000000000..caff0bd6f --- /dev/null +++ b/src/arrow/js/src/vector/timestamp.ts @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { BaseVector } from './base'; +import { Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond } from '../type'; + +/** @ignore */ +export class TimestampVector<T extends Timestamp = Timestamp> extends BaseVector<T> {} +/** @ignore */ +export class TimestampSecondVector extends TimestampVector<TimestampSecond> {} +/** @ignore */ +export class TimestampMillisecondVector extends TimestampVector<TimestampMillisecond> {} +/** @ignore */ +export class TimestampMicrosecondVector extends TimestampVector<TimestampMicrosecond> {} +/** @ignore */ +export class TimestampNanosecondVector extends TimestampVector<TimestampNanosecond> {} diff --git a/src/arrow/js/src/vector/union.ts b/src/arrow/js/src/vector/union.ts new file mode 100644 index 000000000..854519c57 --- /dev/null +++ b/src/arrow/js/src/vector/union.ts @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { BaseVector } from './base'; +import { Union, DenseUnion, SparseUnion} from '../type'; + +/** @ignore */ +export class UnionVector<T extends Union = Union> extends BaseVector<T> { + public get typeIdToChildIndex() { return this.data.type.typeIdToChildIndex; } +} + +/** @ignore */ +export class DenseUnionVector extends UnionVector<DenseUnion> { + public get valueOffsets() { return this.data.valueOffsets!; } +} + +/** @ignore */ +export class SparseUnionVector extends UnionVector<SparseUnion> {} diff --git a/src/arrow/js/src/vector/utf8.ts b/src/arrow/js/src/vector/utf8.ts new file mode 100644 index 000000000..a891c0dc5 --- /dev/null +++ b/src/arrow/js/src/vector/utf8.ts @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Vector } from '../vector'; +import { Chunked } from './chunked'; +import { BaseVector } from './base'; +import { Binary, Utf8 } from '../type'; +import { VectorBuilderOptions } from './index'; +import { vectorFromValuesWithType } from './index'; +import { VectorBuilderOptionsAsync } from './index'; + +/** @ignore */ +export class Utf8Vector extends BaseVector<Utf8> { + public static from<TNull = any>(input: Iterable<string | TNull>): Utf8Vector; + public static from<TNull = any>(input: AsyncIterable<string | TNull>): Promise<Utf8Vector>; + public static from<TNull = any>(input: VectorBuilderOptions<Utf8, string | TNull>): Chunked<Utf8>; + public static from<TNull = any>(input: VectorBuilderOptionsAsync<Utf8, string | TNull>): Promise<Chunked<Utf8>>; + /** @nocollapse */ + public static from<TNull = any>(input: Iterable<string | TNull> | AsyncIterable<string | TNull> | VectorBuilderOptions<Utf8, string | TNull> | VectorBuilderOptionsAsync<Utf8, string | TNull>) { + return vectorFromValuesWithType(() => new Utf8(), input); + } + public asBinary() { + return Vector.new(this.data.clone(new Binary())); + } +} diff --git a/src/arrow/js/src/visitor.ts b/src/arrow/js/src/visitor.ts new file mode 100644 index 000000000..3a63c93f9 --- /dev/null +++ b/src/arrow/js/src/visitor.ts @@ -0,0 +1,260 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from './data'; +import { Vector } from './vector'; +import { Type, Precision, DateUnit, TimeUnit, IntervalUnit, UnionMode } from './enum'; +import { DataType, Float, Int, Date_, Interval, Time, Timestamp, Union, } from './type'; + +export abstract class Visitor { + public visitMany(nodes: any[], ...args: any[][]) { + return nodes.map((node, i) => this.visit(node, ...args.map((x) => x[i]))); + } + public visit(...args: any[]) { + return this.getVisitFn(args[0], false).apply(this, args); + } + public getVisitFn(node: any, throwIfNotFound = true) { + return getVisitFn(this, node, throwIfNotFound); + } + public visitNull (_node: any, ..._args: any[]): any { return null; } + public visitBool (_node: any, ..._args: any[]): any { return null; } + public visitInt (_node: any, ..._args: any[]): any { return null; } + public visitFloat (_node: any, ..._args: any[]): any { return null; } + public visitUtf8 (_node: any, ..._args: any[]): any { return null; } + public visitBinary (_node: any, ..._args: any[]): any { return null; } + public visitFixedSizeBinary (_node: any, ..._args: any[]): any { return null; } + public visitDate (_node: any, ..._args: any[]): any { return null; } + public visitTimestamp (_node: any, ..._args: any[]): any { return null; } + public visitTime (_node: any, ..._args: any[]): any { return null; } + public visitDecimal (_node: any, ..._args: any[]): any { return null; } + public visitList (_node: any, ..._args: any[]): any { return null; } + public visitStruct (_node: any, ..._args: any[]): any { return null; } + public visitUnion (_node: any, ..._args: any[]): any { return null; } + public visitDictionary (_node: any, ..._args: any[]): any { return null; } + public visitInterval (_node: any, ..._args: any[]): any { return null; } + public visitFixedSizeList (_node: any, ..._args: any[]): any { return null; } + public visitMap (_node: any, ..._args: any[]): any { return null; } +} + +/** @ignore */ +function getVisitFn<T extends DataType>(visitor: Visitor, node: any, throwIfNotFound = true) { + let fn: any = null; + let dtype: T['TType'] = Type.NONE; + if (node instanceof Data ) dtype = inferDType(node.type as T); + else if (node instanceof Vector ) dtype = inferDType(node.type as T); + else if (node instanceof DataType) dtype = inferDType(node as T); + else if (typeof (dtype = node) !== 'number') dtype = Type[node] as any as T['TType']; + + switch (dtype) { + case Type.Null: fn = visitor.visitNull; break; + case Type.Bool: fn = visitor.visitBool; break; + case Type.Int: fn = visitor.visitInt; break; + case Type.Int8: fn = visitor.visitInt8 || visitor.visitInt; break; + case Type.Int16: fn = visitor.visitInt16 || visitor.visitInt; break; + case Type.Int32: fn = visitor.visitInt32 || visitor.visitInt; break; + case Type.Int64: fn = visitor.visitInt64 || visitor.visitInt; break; + case Type.Uint8: fn = visitor.visitUint8 || visitor.visitInt; break; + case Type.Uint16: fn = visitor.visitUint16 || visitor.visitInt; break; + case Type.Uint32: fn = visitor.visitUint32 || visitor.visitInt; break; + case Type.Uint64: fn = visitor.visitUint64 || visitor.visitInt; break; + case Type.Float: fn = visitor.visitFloat; break; + case Type.Float16: fn = visitor.visitFloat16 || visitor.visitFloat; break; + case Type.Float32: fn = visitor.visitFloat32 || visitor.visitFloat; break; + case Type.Float64: fn = visitor.visitFloat64 || visitor.visitFloat; break; + case Type.Utf8: fn = visitor.visitUtf8; break; + case Type.Binary: fn = visitor.visitBinary; break; + case Type.FixedSizeBinary: fn = visitor.visitFixedSizeBinary; break; + case Type.Date: fn = visitor.visitDate; break; + case Type.DateDay: fn = visitor.visitDateDay || visitor.visitDate; break; + case Type.DateMillisecond: fn = visitor.visitDateMillisecond || visitor.visitDate; break; + case Type.Timestamp: fn = visitor.visitTimestamp; break; + case Type.TimestampSecond: fn = visitor.visitTimestampSecond || visitor.visitTimestamp; break; + case Type.TimestampMillisecond: fn = visitor.visitTimestampMillisecond || visitor.visitTimestamp; break; + case Type.TimestampMicrosecond: fn = visitor.visitTimestampMicrosecond || visitor.visitTimestamp; break; + case Type.TimestampNanosecond: fn = visitor.visitTimestampNanosecond || visitor.visitTimestamp; break; + case Type.Time: fn = visitor.visitTime; break; + case Type.TimeSecond: fn = visitor.visitTimeSecond || visitor.visitTime; break; + case Type.TimeMillisecond: fn = visitor.visitTimeMillisecond || visitor.visitTime; break; + case Type.TimeMicrosecond: fn = visitor.visitTimeMicrosecond || visitor.visitTime; break; + case Type.TimeNanosecond: fn = visitor.visitTimeNanosecond || visitor.visitTime; break; + case Type.Decimal: fn = visitor.visitDecimal; break; + case Type.List: fn = visitor.visitList; break; + case Type.Struct: fn = visitor.visitStruct; break; + case Type.Union: fn = visitor.visitUnion; break; + case Type.DenseUnion: fn = visitor.visitDenseUnion || visitor.visitUnion; break; + case Type.SparseUnion: fn = visitor.visitSparseUnion || visitor.visitUnion; break; + case Type.Dictionary: fn = visitor.visitDictionary; break; + case Type.Interval: fn = visitor.visitInterval; break; + case Type.IntervalDayTime: fn = visitor.visitIntervalDayTime || visitor.visitInterval; break; + case Type.IntervalYearMonth: fn = visitor.visitIntervalYearMonth || visitor.visitInterval; break; + case Type.FixedSizeList: fn = visitor.visitFixedSizeList; break; + case Type.Map: fn = visitor.visitMap; break; + } + if (typeof fn === 'function') return fn; + if (!throwIfNotFound) return () => null; + throw new Error(`Unrecognized type '${Type[dtype]}'`); +} + +/** @ignore */ +function inferDType<T extends DataType>(type: T): Type { + switch (type.typeId) { + case Type.Null: return Type.Null; + case Type.Int: { + const { bitWidth, isSigned } = (type as any as Int); + switch (bitWidth) { + case 8: return isSigned ? Type.Int8 : Type.Uint8 ; + case 16: return isSigned ? Type.Int16 : Type.Uint16; + case 32: return isSigned ? Type.Int32 : Type.Uint32; + case 64: return isSigned ? Type.Int64 : Type.Uint64; + } + // @ts-ignore + return Type.Int; + } + case Type.Float: + switch((type as any as Float).precision) { + case Precision.HALF: return Type.Float16; + case Precision.SINGLE: return Type.Float32; + case Precision.DOUBLE: return Type.Float64; + } + // @ts-ignore + return Type.Float; + case Type.Binary: return Type.Binary; + case Type.Utf8: return Type.Utf8; + case Type.Bool: return Type.Bool; + case Type.Decimal: return Type.Decimal; + case Type.Time: + switch ((type as any as Time).unit) { + case TimeUnit.SECOND: return Type.TimeSecond; + case TimeUnit.MILLISECOND: return Type.TimeMillisecond; + case TimeUnit.MICROSECOND: return Type.TimeMicrosecond; + case TimeUnit.NANOSECOND: return Type.TimeNanosecond; + } + // @ts-ignore + return Type.Time; + case Type.Timestamp: + switch ((type as any as Timestamp).unit) { + case TimeUnit.SECOND: return Type.TimestampSecond; + case TimeUnit.MILLISECOND: return Type.TimestampMillisecond; + case TimeUnit.MICROSECOND: return Type.TimestampMicrosecond; + case TimeUnit.NANOSECOND: return Type.TimestampNanosecond; + } + // @ts-ignore + return Type.Timestamp; + case Type.Date: + switch ((type as any as Date_).unit) { + case DateUnit.DAY: return Type.DateDay; + case DateUnit.MILLISECOND: return Type.DateMillisecond; + } + // @ts-ignore + return Type.Date; + case Type.Interval: + switch ((type as any as Interval).unit) { + case IntervalUnit.DAY_TIME: return Type.IntervalDayTime; + case IntervalUnit.YEAR_MONTH: return Type.IntervalYearMonth; + } + // @ts-ignore + return Type.Interval; + case Type.Map: return Type.Map; + case Type.List: return Type.List; + case Type.Struct: return Type.Struct; + case Type.Union: + switch ((type as any as Union).mode) { + case UnionMode.Dense: return Type.DenseUnion; + case UnionMode.Sparse: return Type.SparseUnion; + } + // @ts-ignore + return Type.Union; + case Type.FixedSizeBinary: return Type.FixedSizeBinary; + case Type.FixedSizeList: return Type.FixedSizeList; + case Type.Dictionary: return Type.Dictionary; + } + throw new Error(`Unrecognized type '${Type[type.typeId]}'`); +} + +export interface Visitor { + visitNull (node: any, ...args: any[]): any; + visitBool (node: any, ...args: any[]): any; + visitInt (node: any, ...args: any[]): any; + visitInt8? (node: any, ...args: any[]): any; + visitInt16? (node: any, ...args: any[]): any; + visitInt32? (node: any, ...args: any[]): any; + visitInt64? (node: any, ...args: any[]): any; + visitUint8? (node: any, ...args: any[]): any; + visitUint16? (node: any, ...args: any[]): any; + visitUint32? (node: any, ...args: any[]): any; + visitUint64? (node: any, ...args: any[]): any; + visitFloat (node: any, ...args: any[]): any; + visitFloat16? (node: any, ...args: any[]): any; + visitFloat32? (node: any, ...args: any[]): any; + visitFloat64? (node: any, ...args: any[]): any; + visitUtf8 (node: any, ...args: any[]): any; + visitBinary (node: any, ...args: any[]): any; + visitFixedSizeBinary (node: any, ...args: any[]): any; + visitDate (node: any, ...args: any[]): any; + visitDateDay? (node: any, ...args: any[]): any; + visitDateMillisecond? (node: any, ...args: any[]): any; + visitTimestamp (node: any, ...args: any[]): any; + visitTimestampSecond? (node: any, ...args: any[]): any; + visitTimestampMillisecond? (node: any, ...args: any[]): any; + visitTimestampMicrosecond? (node: any, ...args: any[]): any; + visitTimestampNanosecond? (node: any, ...args: any[]): any; + visitTime (node: any, ...args: any[]): any; + visitTimeSecond? (node: any, ...args: any[]): any; + visitTimeMillisecond? (node: any, ...args: any[]): any; + visitTimeMicrosecond? (node: any, ...args: any[]): any; + visitTimeNanosecond? (node: any, ...args: any[]): any; + visitDecimal (node: any, ...args: any[]): any; + visitList (node: any, ...args: any[]): any; + visitStruct (node: any, ...args: any[]): any; + visitUnion (node: any, ...args: any[]): any; + visitDenseUnion? (node: any, ...args: any[]): any; + visitSparseUnion? (node: any, ...args: any[]): any; + visitDictionary (node: any, ...args: any[]): any; + visitInterval (node: any, ...args: any[]): any; + visitIntervalDayTime? (node: any, ...args: any[]): any; + visitIntervalYearMonth? (node: any, ...args: any[]): any; + visitFixedSizeList (node: any, ...args: any[]): any; + visitMap (node: any, ...args: any[]): any; +} + +// Add these here so they're picked up by the externs creator +// in the build, and closure-compiler doesn't minify them away +(Visitor.prototype as any).visitInt8 = null; +(Visitor.prototype as any).visitInt16 = null; +(Visitor.prototype as any).visitInt32 = null; +(Visitor.prototype as any).visitInt64 = null; +(Visitor.prototype as any).visitUint8 = null; +(Visitor.prototype as any).visitUint16 = null; +(Visitor.prototype as any).visitUint32 = null; +(Visitor.prototype as any).visitUint64 = null; +(Visitor.prototype as any).visitFloat16 = null; +(Visitor.prototype as any).visitFloat32 = null; +(Visitor.prototype as any).visitFloat64 = null; +(Visitor.prototype as any).visitDateDay = null; +(Visitor.prototype as any).visitDateMillisecond = null; +(Visitor.prototype as any).visitTimestampSecond = null; +(Visitor.prototype as any).visitTimestampMillisecond = null; +(Visitor.prototype as any).visitTimestampMicrosecond = null; +(Visitor.prototype as any).visitTimestampNanosecond = null; +(Visitor.prototype as any).visitTimeSecond = null; +(Visitor.prototype as any).visitTimeMillisecond = null; +(Visitor.prototype as any).visitTimeMicrosecond = null; +(Visitor.prototype as any).visitTimeNanosecond = null; +(Visitor.prototype as any).visitDenseUnion = null; +(Visitor.prototype as any).visitSparseUnion = null; +(Visitor.prototype as any).visitIntervalDayTime = null; +(Visitor.prototype as any).visitIntervalYearMonth = null; diff --git a/src/arrow/js/src/visitor/builderctor.ts b/src/arrow/js/src/visitor/builderctor.ts new file mode 100644 index 000000000..ac35a9874 --- /dev/null +++ b/src/arrow/js/src/visitor/builderctor.ts @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import { Type } from '../enum'; +import { DataType } from '../type'; +import { Visitor } from '../visitor'; +import { VectorType, BuilderCtor } from '../interfaces'; +import { BinaryBuilder } from '../builder/binary'; +import { BoolBuilder } from '../builder/bool'; +import { DateBuilder, DateDayBuilder, DateMillisecondBuilder } from '../builder/date'; +import { DecimalBuilder } from '../builder/decimal'; +import { DictionaryBuilder } from '../builder/dictionary'; +import { FixedSizeBinaryBuilder } from '../builder/fixedsizebinary'; +import { FixedSizeListBuilder } from '../builder/fixedsizelist'; +import { FloatBuilder, Float16Builder, Float32Builder, Float64Builder } from '../builder/float'; +import { IntervalBuilder, IntervalDayTimeBuilder, IntervalYearMonthBuilder } from '../builder/interval'; +import { IntBuilder, Int8Builder, Int16Builder, Int32Builder, Int64Builder, Uint8Builder, Uint16Builder, Uint32Builder, Uint64Builder } from '../builder/int'; +import { ListBuilder } from '../builder/list'; +import { MapBuilder } from '../builder/map'; +import { NullBuilder } from '../builder/null'; +import { StructBuilder } from '../builder/struct'; +import { TimestampBuilder, TimestampSecondBuilder, TimestampMillisecondBuilder, TimestampMicrosecondBuilder, TimestampNanosecondBuilder } from '../builder/timestamp'; +import { TimeBuilder, TimeSecondBuilder, TimeMillisecondBuilder, TimeMicrosecondBuilder, TimeNanosecondBuilder } from '../builder/time'; +import { UnionBuilder, DenseUnionBuilder, SparseUnionBuilder } from '../builder/union'; +import { Utf8Builder } from '../builder/utf8'; + +/** @ignore */ +export interface GetBuilderCtor extends Visitor { + visit<T extends Type>(type: T): BuilderCtor<T>; + visitMany<T extends Type>(types: T[]): BuilderCtor<T>[]; + getVisitFn<T extends Type>(type: T): () => BuilderCtor<T>; + getVisitFn<T extends DataType>(node: VectorType<T> | Data<T> | T): () => BuilderCtor<T>; +} + +/** @ignore */ +export class GetBuilderCtor extends Visitor { + public visitNull () { return NullBuilder; } + public visitBool () { return BoolBuilder; } + public visitInt () { return IntBuilder; } + public visitInt8 () { return Int8Builder; } + public visitInt16 () { return Int16Builder; } + public visitInt32 () { return Int32Builder; } + public visitInt64 () { return Int64Builder; } + public visitUint8 () { return Uint8Builder; } + public visitUint16 () { return Uint16Builder; } + public visitUint32 () { return Uint32Builder; } + public visitUint64 () { return Uint64Builder; } + public visitFloat () { return FloatBuilder; } + public visitFloat16 () { return Float16Builder; } + public visitFloat32 () { return Float32Builder; } + public visitFloat64 () { return Float64Builder; } + public visitUtf8 () { return Utf8Builder; } + public visitBinary () { return BinaryBuilder; } + public visitFixedSizeBinary () { return FixedSizeBinaryBuilder; } + public visitDate () { return DateBuilder; } + public visitDateDay () { return DateDayBuilder; } + public visitDateMillisecond () { return DateMillisecondBuilder; } + public visitTimestamp () { return TimestampBuilder; } + public visitTimestampSecond () { return TimestampSecondBuilder; } + public visitTimestampMillisecond () { return TimestampMillisecondBuilder; } + public visitTimestampMicrosecond () { return TimestampMicrosecondBuilder; } + public visitTimestampNanosecond () { return TimestampNanosecondBuilder; } + public visitTime () { return TimeBuilder; } + public visitTimeSecond () { return TimeSecondBuilder; } + public visitTimeMillisecond () { return TimeMillisecondBuilder; } + public visitTimeMicrosecond () { return TimeMicrosecondBuilder; } + public visitTimeNanosecond () { return TimeNanosecondBuilder; } + public visitDecimal () { return DecimalBuilder; } + public visitList () { return ListBuilder; } + public visitStruct () { return StructBuilder; } + public visitUnion () { return UnionBuilder; } + public visitDenseUnion () { return DenseUnionBuilder; } + public visitSparseUnion () { return SparseUnionBuilder; } + public visitDictionary () { return DictionaryBuilder; } + public visitInterval () { return IntervalBuilder; } + public visitIntervalDayTime () { return IntervalDayTimeBuilder; } + public visitIntervalYearMonth () { return IntervalYearMonthBuilder; } + public visitFixedSizeList () { return FixedSizeListBuilder; } + public visitMap () { return MapBuilder; } +} + +/** @ignore */ +export const instance = new GetBuilderCtor(); diff --git a/src/arrow/js/src/visitor/bytewidth.ts b/src/arrow/js/src/visitor/bytewidth.ts new file mode 100644 index 000000000..8be7c7a64 --- /dev/null +++ b/src/arrow/js/src/visitor/bytewidth.ts @@ -0,0 +1,68 @@ +/* istanbul ignore file */ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import { Visitor } from '../visitor'; +import { VectorType } from '../interfaces'; +import { Type, TimeUnit } from '../enum'; +import { Schema, Field } from '../schema'; +import { + DataType, Dictionary, + Float, Int, Date_, Interval, Time, Timestamp, + Bool, Null, Utf8, Binary, Decimal, FixedSizeBinary, + List, FixedSizeList, Map_, Struct, Union, +} from '../type'; + +/** @ignore */ const sum = (x: number, y: number) => x + y; +/** @ignore */ const variableWidthColumnErrorMessage = (type: DataType) => `Cannot compute the byte width of variable-width column ${type}`; + +/** @ignore */ +export interface ByteWidthVisitor extends Visitor { + visit<T extends DataType>(node: T): number; + visitMany<T extends DataType>(nodes: T[]): number[]; + getVisitFn<T extends Type> (node: T): (type: DataType<T>) => number; + getVisitFn<T extends DataType>(node: VectorType<T> | Data<T> | T): (type: T) => number; +} + +/** @ignore */ +export class ByteWidthVisitor extends Visitor { + public visitNull (____: Null ) { return 0; } + public visitInt (type: Int ) { return type.bitWidth / 8; } + public visitFloat (type: Float ) { return type.ArrayType.BYTES_PER_ELEMENT; } + public visitBinary (type: Binary ) { throw new Error(variableWidthColumnErrorMessage(type)); } + public visitUtf8 (type: Utf8 ) { throw new Error(variableWidthColumnErrorMessage(type)); } + public visitBool (____: Bool ) { return 1 / 8; } + public visitDecimal (____: Decimal ) { return 16; } + public visitDate (type: Date_ ) { return (type.unit + 1) * 4; } + public visitTime (type: Time ) { return type.bitWidth / 8; } + public visitTimestamp (type: Timestamp ) { return type.unit === TimeUnit.SECOND ? 4 : 8; } + public visitInterval (type: Interval ) { return (type.unit + 1) * 4; } + public visitList (type: List ) { throw new Error(variableWidthColumnErrorMessage(type)); } + public visitStruct (type: Struct ) { return this.visitFields(type.children).reduce(sum, 0); } + public visitUnion (type: Union ) { return this.visitFields(type.children).reduce(sum, 0); } + public visitFixedSizeBinary (type: FixedSizeBinary ) { return type.byteWidth; } + public visitFixedSizeList (type: FixedSizeList ) { return type.listSize * this.visitFields(type.children).reduce(sum, 0); } + public visitMap (type: Map_ ) { return this.visitFields(type.children).reduce(sum, 0); } + public visitDictionary (type: Dictionary ) { return this.visit(type.indices); } + public visitFields (fields: Field[] ) { return (fields || []).map((field) => this.visit(field.type)); } + public visitSchema (schema: Schema ) { return this.visitFields(schema.fields).reduce(sum, 0); } +} + +/** @ignore */ +export const instance = new ByteWidthVisitor(); diff --git a/src/arrow/js/src/visitor/get.ts b/src/arrow/js/src/visitor/get.ts new file mode 100644 index 000000000..733418c0a --- /dev/null +++ b/src/arrow/js/src/visitor/get.ts @@ -0,0 +1,321 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import { BN } from '../util/bn'; +import { Visitor } from '../visitor'; +import { decodeUtf8 } from '../util/utf8'; +import { VectorType } from '../interfaces'; +import { uint16ToFloat64 } from '../util/math'; +import { Type, UnionMode, Precision, DateUnit, TimeUnit, IntervalUnit } from '../enum'; +import { + DataType, Dictionary, + Bool, Null, Utf8, Binary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, + Float, Float16, Float32, Float64, + Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, + Date_, DateDay, DateMillisecond, + Interval, IntervalDayTime, IntervalYearMonth, + Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, + Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, + Union, DenseUnion, SparseUnion, +} from '../type'; + +/** @ignore */ +export interface GetVisitor extends Visitor { + visit<T extends VectorType> (node: T, index: number): T['TValue']; + visitMany<T extends VectorType> (nodes: T[], indices: number[]): T['TValue'][]; + getVisitFn<T extends Type> (node: T): (vector: VectorType<T>, index: number) => VectorType<T>['TValue']; + getVisitFn<T extends DataType>(node: VectorType<T> | Data<T> | T): (vector: VectorType<T>, index: number) => VectorType<T>['TValue']; + visitNull <T extends Null> (vector: VectorType<T>, index: number): T['TValue']; + visitBool <T extends Bool> (vector: VectorType<T>, index: number): T['TValue']; + visitInt <T extends Int> (vector: VectorType<T>, index: number): T['TValue']; + visitInt8 <T extends Int8> (vector: VectorType<T>, index: number): T['TValue']; + visitInt16 <T extends Int16> (vector: VectorType<T>, index: number): T['TValue']; + visitInt32 <T extends Int32> (vector: VectorType<T>, index: number): T['TValue']; + visitInt64 <T extends Int64> (vector: VectorType<T>, index: number): T['TValue']; + visitUint8 <T extends Uint8> (vector: VectorType<T>, index: number): T['TValue']; + visitUint16 <T extends Uint16> (vector: VectorType<T>, index: number): T['TValue']; + visitUint32 <T extends Uint32> (vector: VectorType<T>, index: number): T['TValue']; + visitUint64 <T extends Uint64> (vector: VectorType<T>, index: number): T['TValue']; + visitFloat <T extends Float> (vector: VectorType<T>, index: number): T['TValue']; + visitFloat16 <T extends Float16> (vector: VectorType<T>, index: number): T['TValue']; + visitFloat32 <T extends Float32> (vector: VectorType<T>, index: number): T['TValue']; + visitFloat64 <T extends Float64> (vector: VectorType<T>, index: number): T['TValue']; + visitUtf8 <T extends Utf8> (vector: VectorType<T>, index: number): T['TValue']; + visitBinary <T extends Binary> (vector: VectorType<T>, index: number): T['TValue']; + visitFixedSizeBinary <T extends FixedSizeBinary> (vector: VectorType<T>, index: number): T['TValue']; + visitDate <T extends Date_> (vector: VectorType<T>, index: number): T['TValue']; + visitDateDay <T extends DateDay> (vector: VectorType<T>, index: number): T['TValue']; + visitDateMillisecond <T extends DateMillisecond> (vector: VectorType<T>, index: number): T['TValue']; + visitTimestamp <T extends Timestamp> (vector: VectorType<T>, index: number): T['TValue']; + visitTimestampSecond <T extends TimestampSecond> (vector: VectorType<T>, index: number): T['TValue']; + visitTimestampMillisecond <T extends TimestampMillisecond> (vector: VectorType<T>, index: number): T['TValue']; + visitTimestampMicrosecond <T extends TimestampMicrosecond> (vector: VectorType<T>, index: number): T['TValue']; + visitTimestampNanosecond <T extends TimestampNanosecond> (vector: VectorType<T>, index: number): T['TValue']; + visitTime <T extends Time> (vector: VectorType<T>, index: number): T['TValue']; + visitTimeSecond <T extends TimeSecond> (vector: VectorType<T>, index: number): T['TValue']; + visitTimeMillisecond <T extends TimeMillisecond> (vector: VectorType<T>, index: number): T['TValue']; + visitTimeMicrosecond <T extends TimeMicrosecond> (vector: VectorType<T>, index: number): T['TValue']; + visitTimeNanosecond <T extends TimeNanosecond> (vector: VectorType<T>, index: number): T['TValue']; + visitDecimal <T extends Decimal> (vector: VectorType<T>, index: number): T['TValue']; + visitList <T extends List> (vector: VectorType<T>, index: number): T['TValue']; + visitStruct <T extends Struct> (vector: VectorType<T>, index: number): T['TValue']; + visitUnion <T extends Union> (vector: VectorType<T>, index: number): T['TValue']; + visitDenseUnion <T extends DenseUnion> (vector: VectorType<T>, index: number): T['TValue']; + visitSparseUnion <T extends SparseUnion> (vector: VectorType<T>, index: number): T['TValue']; + visitDictionary <T extends Dictionary> (vector: VectorType<T>, index: number): T['TValue']; + visitInterval <T extends Interval> (vector: VectorType<T>, index: number): T['TValue']; + visitIntervalDayTime <T extends IntervalDayTime> (vector: VectorType<T>, index: number): T['TValue']; + visitIntervalYearMonth <T extends IntervalYearMonth> (vector: VectorType<T>, index: number): T['TValue']; + visitFixedSizeList <T extends FixedSizeList> (vector: VectorType<T>, index: number): T['TValue']; + visitMap <T extends Map_> (vector: VectorType<T>, index: number): T['TValue']; +} + +/** @ignore */ +export class GetVisitor extends Visitor {} + +/** @ignore */const epochDaysToMs = (data: Int32Array, index: number) => 86400000 * data[index]; +/** @ignore */const epochMillisecondsLongToMs = (data: Int32Array, index: number) => 4294967296 * (data[index + 1]) + (data[index] >>> 0); +/** @ignore */const epochMicrosecondsLongToMs = (data: Int32Array, index: number) => 4294967296 * (data[index + 1] / 1000) + ((data[index] >>> 0) / 1000); +/** @ignore */const epochNanosecondsLongToMs = (data: Int32Array, index: number) => 4294967296 * (data[index + 1] / 1000000) + ((data[index] >>> 0) / 1000000); + +/** @ignore */const epochMillisecondsToDate = (epochMs: number) => new Date(epochMs); +/** @ignore */const epochDaysToDate = (data: Int32Array, index: number) => epochMillisecondsToDate(epochDaysToMs(data, index)); +/** @ignore */const epochMillisecondsLongToDate = (data: Int32Array, index: number) => epochMillisecondsToDate(epochMillisecondsLongToMs(data, index)); + +/** @ignore */ +const getNull = <T extends Null>(_vector: VectorType<T>, _index: number): T['TValue'] => null; +/** @ignore */ +const getVariableWidthBytes = (values: Uint8Array, valueOffsets: Int32Array, index: number) => { + const { [index]: x, [index + 1]: y } = valueOffsets; + return x != null && y != null ? values.subarray(x, y) : null as any; +}; + +/** @ignore */ +const getBool = <T extends Bool>({ offset, values }: VectorType<T>, index: number): T['TValue'] => { + const idx = offset + index; + const byte = values[idx >> 3]; + return (byte & 1 << (idx % 8)) !== 0; +}; + +/** @ignore */ +type Numeric1X = Int8 | Int16 | Int32 | Uint8 | Uint16 | Uint32 | Float32 | Float64; +/** @ignore */ +type Numeric2X = Int64 | Uint64; + +/** @ignore */ +const getDateDay = <T extends DateDay> ({ values }: VectorType<T>, index: number): T['TValue'] => epochDaysToDate(values, index); +/** @ignore */ +const getDateMillisecond = <T extends DateMillisecond>({ values }: VectorType<T>, index: number): T['TValue'] => epochMillisecondsLongToDate(values, index * 2); +/** @ignore */ +const getNumeric = <T extends Numeric1X> ({ stride, values }: VectorType<T>, index: number): T['TValue'] => values[stride * index]; +/** @ignore */ +const getFloat16 = <T extends Float16> ({ stride, values }: VectorType<T>, index: number): T['TValue'] => uint16ToFloat64(values[stride * index]); +/** @ignore */ +const getBigInts = <T extends Numeric2X>({ stride, values, type }: VectorType<T>, index: number): T['TValue'] => <any> BN.new(values.subarray(stride * index, stride * (index + 1)), type.isSigned); +/** @ignore */ +const getFixedSizeBinary = <T extends FixedSizeBinary>({ stride, values }: VectorType<T>, index: number): T['TValue'] => values.subarray(stride * index, stride * (index + 1)); + +/** @ignore */ +const getBinary = <T extends Binary>({ values, valueOffsets }: VectorType<T>, index: number): T['TValue'] => getVariableWidthBytes(values, valueOffsets, index); +/** @ignore */ +const getUtf8 = <T extends Utf8>({ values, valueOffsets }: VectorType<T>, index: number): T['TValue'] => { + const bytes = getVariableWidthBytes(values, valueOffsets, index); + return bytes !== null ? decodeUtf8(bytes) : null as any; +}; + +/* istanbul ignore next */ +/** @ignore */ +const getInt = <T extends Int>(vector: VectorType<T>, index: number): T['TValue'] => ( + vector.type.bitWidth < 64 + ? getNumeric(vector as VectorType<Numeric1X>, index) + : getBigInts(vector as VectorType<Numeric2X>, index) +); + +/* istanbul ignore next */ +/** @ignore */ +const getFloat = <T extends Float> (vector: VectorType<T>, index: number): T['TValue'] => ( + vector.type.precision !== Precision.HALF + ? getNumeric(vector as VectorType<Numeric1X>, index) + : getFloat16(vector as VectorType<Float16>, index) +); + +/* istanbul ignore next */ +/** @ignore */ +const getDate = <T extends Date_> (vector: VectorType<T>, index: number): T['TValue'] => ( + vector.type.unit === DateUnit.DAY + ? getDateDay(vector as VectorType<DateDay>, index) + : getDateMillisecond(vector as VectorType<DateMillisecond>, index) +); + +/** @ignore */ +const getTimestampSecond = <T extends TimestampSecond> ({ values }: VectorType<T>, index: number): T['TValue'] => 1000 * epochMillisecondsLongToMs(values, index * 2); +/** @ignore */ +const getTimestampMillisecond = <T extends TimestampMillisecond>({ values }: VectorType<T>, index: number): T['TValue'] => epochMillisecondsLongToMs(values, index * 2); +/** @ignore */ +const getTimestampMicrosecond = <T extends TimestampMicrosecond>({ values }: VectorType<T>, index: number): T['TValue'] => epochMicrosecondsLongToMs(values, index * 2); +/** @ignore */ +const getTimestampNanosecond = <T extends TimestampNanosecond> ({ values }: VectorType<T>, index: number): T['TValue'] => epochNanosecondsLongToMs(values, index * 2); +/* istanbul ignore next */ +/** @ignore */ +const getTimestamp = <T extends Timestamp>(vector: VectorType<T>, index: number): T['TValue'] => { + switch (vector.type.unit) { + case TimeUnit.SECOND: return getTimestampSecond(vector as VectorType<TimestampSecond>, index); + case TimeUnit.MILLISECOND: return getTimestampMillisecond(vector as VectorType<TimestampMillisecond>, index); + case TimeUnit.MICROSECOND: return getTimestampMicrosecond(vector as VectorType<TimestampMicrosecond>, index); + case TimeUnit.NANOSECOND: return getTimestampNanosecond(vector as VectorType<TimestampNanosecond>, index); + } +}; + +/** @ignore */ +const getTimeSecond = <T extends TimeSecond> ({ values, stride }: VectorType<T>, index: number): T['TValue'] => values[stride * index]; +/** @ignore */ +const getTimeMillisecond = <T extends TimeMillisecond>({ values, stride }: VectorType<T>, index: number): T['TValue'] => values[stride * index]; +/** @ignore */ +const getTimeMicrosecond = <T extends TimeMicrosecond>({ values }: VectorType<T>, index: number): T['TValue'] => BN.signed(values.subarray(2 * index, 2 * (index + 1))); +/** @ignore */ +const getTimeNanosecond = <T extends TimeNanosecond> ({ values }: VectorType<T>, index: number): T['TValue'] => BN.signed(values.subarray(2 * index, 2 * (index + 1))); +/* istanbul ignore next */ +/** @ignore */ +const getTime = <T extends Time>(vector: VectorType<T>, index: number): T['TValue'] => { + switch (vector.type.unit) { + case TimeUnit.SECOND: return getTimeSecond(vector as VectorType<TimeSecond>, index); + case TimeUnit.MILLISECOND: return getTimeMillisecond(vector as VectorType<TimeMillisecond>, index); + case TimeUnit.MICROSECOND: return getTimeMicrosecond(vector as VectorType<TimeMicrosecond>, index); + case TimeUnit.NANOSECOND: return getTimeNanosecond(vector as VectorType<TimeNanosecond>, index); + } +}; + +/** @ignore */ +const getDecimal = <T extends Decimal>({ values }: VectorType<T>, index: number): T['TValue'] => BN.decimal(values.subarray(4 * index, 4 * (index + 1))); + +/** @ignore */ +const getList = <T extends List>(vector: VectorType<T>, index: number): T['TValue'] => { + const child = vector.getChildAt(0)!, { valueOffsets, stride } = vector; + return child.slice(valueOffsets[index * stride], valueOffsets[(index * stride) + 1]) as T['TValue']; +}; + +/** @ignore */ +const getMap = <T extends Map_>(vector: VectorType<T>, index: number): T['TValue'] => { + return vector.bind(index) as T['TValue']; +}; + +/** @ignore */ +const getStruct = <T extends Struct>(vector: VectorType<T>, index: number): T['TValue'] => { + return vector.bind(index) as T['TValue']; +}; + +/* istanbul ignore next */ +/** @ignore */ +const getUnion = < + V extends VectorType<Union> | VectorType<DenseUnion> | VectorType<SparseUnion> +>(vector: V, index: number): V['TValue'] => { + return vector.type.mode === UnionMode.Dense ? + getDenseUnion(vector as VectorType<DenseUnion>, index) : + getSparseUnion(vector as VectorType<SparseUnion>, index); +}; + +/** @ignore */ +const getDenseUnion = <T extends DenseUnion>(vector: VectorType<T>, index: number): T['TValue'] => { + const childIndex = vector.typeIdToChildIndex[vector.typeIds[index]]; + const child = vector.getChildAt(childIndex); + return child ? child.get(vector.valueOffsets[index]) : null; +}; + +/** @ignore */ +const getSparseUnion = <T extends SparseUnion>(vector: VectorType<T>, index: number): T['TValue'] => { + const childIndex = vector.typeIdToChildIndex[vector.typeIds[index]]; + const child = vector.getChildAt(childIndex); + return child ? child.get(index) : null; +}; + +/** @ignore */ +const getDictionary = <T extends Dictionary>(vector: VectorType<T>, index: number): T['TValue'] => { + return vector.getValue(vector.getKey(index)!); +}; + +/* istanbul ignore next */ +/** @ignore */ +const getInterval = <T extends Interval>(vector: VectorType<T>, index: number): T['TValue'] => + (vector.type.unit === IntervalUnit.DAY_TIME) + ? getIntervalDayTime(vector as VectorType<IntervalDayTime>, index) + : getIntervalYearMonth(vector as VectorType<IntervalYearMonth>, index); + +/** @ignore */ +const getIntervalDayTime = <T extends IntervalDayTime>({ values }: VectorType<T>, index: number): T['TValue'] => values.subarray(2 * index, 2 * (index + 1)); + +/** @ignore */ +const getIntervalYearMonth = <T extends IntervalYearMonth>({ values }: VectorType<T>, index: number): T['TValue'] => { + const interval = values[index]; + const int32s = new Int32Array(2); + int32s[0] = interval / 12 | 0; /* years */ + int32s[1] = interval % 12 | 0; /* months */ + return int32s; +}; + +/** @ignore */ +const getFixedSizeList = <T extends FixedSizeList>(vector: VectorType<T>, index: number): T['TValue'] => { + const child = vector.getChildAt(0)!, { stride } = vector; + return child.slice(index * stride, (index + 1) * stride) as T['TValue']; +}; + +GetVisitor.prototype.visitNull = getNull; +GetVisitor.prototype.visitBool = getBool; +GetVisitor.prototype.visitInt = getInt; +GetVisitor.prototype.visitInt8 = getNumeric; +GetVisitor.prototype.visitInt16 = getNumeric; +GetVisitor.prototype.visitInt32 = getNumeric; +GetVisitor.prototype.visitInt64 = getBigInts; +GetVisitor.prototype.visitUint8 = getNumeric; +GetVisitor.prototype.visitUint16 = getNumeric; +GetVisitor.prototype.visitUint32 = getNumeric; +GetVisitor.prototype.visitUint64 = getBigInts; +GetVisitor.prototype.visitFloat = getFloat; +GetVisitor.prototype.visitFloat16 = getFloat16; +GetVisitor.prototype.visitFloat32 = getNumeric; +GetVisitor.prototype.visitFloat64 = getNumeric; +GetVisitor.prototype.visitUtf8 = getUtf8; +GetVisitor.prototype.visitBinary = getBinary; +GetVisitor.prototype.visitFixedSizeBinary = getFixedSizeBinary; +GetVisitor.prototype.visitDate = getDate; +GetVisitor.prototype.visitDateDay = getDateDay; +GetVisitor.prototype.visitDateMillisecond = getDateMillisecond; +GetVisitor.prototype.visitTimestamp = getTimestamp; +GetVisitor.prototype.visitTimestampSecond = getTimestampSecond; +GetVisitor.prototype.visitTimestampMillisecond = getTimestampMillisecond; +GetVisitor.prototype.visitTimestampMicrosecond = getTimestampMicrosecond; +GetVisitor.prototype.visitTimestampNanosecond = getTimestampNanosecond; +GetVisitor.prototype.visitTime = getTime; +GetVisitor.prototype.visitTimeSecond = getTimeSecond; +GetVisitor.prototype.visitTimeMillisecond = getTimeMillisecond; +GetVisitor.prototype.visitTimeMicrosecond = getTimeMicrosecond; +GetVisitor.prototype.visitTimeNanosecond = getTimeNanosecond; +GetVisitor.prototype.visitDecimal = getDecimal; +GetVisitor.prototype.visitList = getList; +GetVisitor.prototype.visitStruct = getStruct; +GetVisitor.prototype.visitUnion = getUnion; +GetVisitor.prototype.visitDenseUnion = getDenseUnion; +GetVisitor.prototype.visitSparseUnion = getSparseUnion; +GetVisitor.prototype.visitDictionary = getDictionary; +GetVisitor.prototype.visitInterval = getInterval; +GetVisitor.prototype.visitIntervalDayTime = getIntervalDayTime; +GetVisitor.prototype.visitIntervalYearMonth = getIntervalYearMonth; +GetVisitor.prototype.visitFixedSizeList = getFixedSizeList; +GetVisitor.prototype.visitMap = getMap; + +/** @ignore */ +export const instance = new GetVisitor(); diff --git a/src/arrow/js/src/visitor/indexof.ts b/src/arrow/js/src/visitor/indexof.ts new file mode 100644 index 000000000..ab4678aed --- /dev/null +++ b/src/arrow/js/src/visitor/indexof.ts @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import { Type } from '../enum'; +import { Visitor } from '../visitor'; +import { VectorType } from '../interfaces'; +import { getBool, BitIterator } from '../util/bit'; +import { createElementComparator } from '../util/vector'; +import { + DataType, Dictionary, + Bool, Null, Utf8, Binary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, + Float, Float16, Float32, Float64, + Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, + Date_, DateDay, DateMillisecond, + Interval, IntervalDayTime, IntervalYearMonth, + Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, + Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, + Union, DenseUnion, SparseUnion, +} from '../type'; + +/** @ignore */ +export interface IndexOfVisitor extends Visitor { + visit<T extends VectorType> (node: T, value: T['TValue'] | null, index?: number): number; + visitMany <T extends VectorType> (nodes: T[], values: (T['TValue'] | null)[], indices: (number | undefined)[]): number[]; + getVisitFn<T extends Type> (node: T): (vector: VectorType<T>, value: VectorType<T>['TValue'] | null, index?: number) => number; + getVisitFn<T extends DataType>(node: VectorType<T> | Data<T> | T): (vector: VectorType<T>, value: T['TValue'] | null, index?: number) => number; + visitNull <T extends Null> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitBool <T extends Bool> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitInt <T extends Int> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitInt8 <T extends Int8> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitInt16 <T extends Int16> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitInt32 <T extends Int32> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitInt64 <T extends Int64> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitUint8 <T extends Uint8> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitUint16 <T extends Uint16> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitUint32 <T extends Uint32> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitUint64 <T extends Uint64> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitFloat <T extends Float> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitFloat16 <T extends Float16> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitFloat32 <T extends Float32> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitFloat64 <T extends Float64> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitUtf8 <T extends Utf8> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitBinary <T extends Binary> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitFixedSizeBinary <T extends FixedSizeBinary> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitDate <T extends Date_> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitDateDay <T extends DateDay> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitDateMillisecond <T extends DateMillisecond> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitTimestamp <T extends Timestamp> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitTimestampSecond <T extends TimestampSecond> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitTimestampMillisecond <T extends TimestampMillisecond>(vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitTimestampMicrosecond <T extends TimestampMicrosecond>(vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitTimestampNanosecond <T extends TimestampNanosecond> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitTime <T extends Time> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitTimeSecond <T extends TimeSecond> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitTimeMillisecond <T extends TimeMillisecond> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitTimeMicrosecond <T extends TimeMicrosecond> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitTimeNanosecond <T extends TimeNanosecond> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitDecimal <T extends Decimal> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitList <T extends List> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitStruct <T extends Struct> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitUnion <T extends Union> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitDenseUnion <T extends DenseUnion> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitSparseUnion <T extends SparseUnion> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitDictionary <T extends Dictionary> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitInterval <T extends Interval> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitIntervalDayTime <T extends IntervalDayTime> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitIntervalYearMonth <T extends IntervalYearMonth> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitFixedSizeList <T extends FixedSizeList> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; + visitMap <T extends Map_> (vector: VectorType<T>, value: T['TValue'] | null, index?: number): number; +} + +/** @ignore */ +export class IndexOfVisitor extends Visitor {} + +/** @ignore */ +function nullIndexOf(vector: VectorType<Null>, searchElement?: null) { + // if you're looking for nulls and the vector isn't empty, we've got 'em! + return searchElement === null && vector.length > 0 ? 0 : -1; +} + +/** @ignore */ +function indexOfNull<T extends DataType>(vector: VectorType<T>, fromIndex?: number): number { + const { nullBitmap } = vector.data; + if (!nullBitmap || vector.nullCount <= 0) { + return -1; + } + let i = 0; + for (const isValid of new BitIterator(nullBitmap, vector.data.offset + (fromIndex || 0), vector.length, nullBitmap, getBool)) { + if (!isValid) { return i; } + ++i; + } + return -1; +} + +/** @ignore */ +function indexOfValue<T extends DataType>(vector: VectorType<T>, searchElement?: T['TValue'] | null, fromIndex?: number): number { + if (searchElement === undefined) { return -1; } + if (searchElement === null) { return indexOfNull(vector, fromIndex); } + const compare = createElementComparator(searchElement); + for (let i = (fromIndex || 0) - 1, n = vector.length; ++i < n;) { + if (compare(vector.get(i))) { + return i; + } + } + return -1; +} + +/** @ignore */ +function indexOfUnion<T extends DataType>(vector: VectorType<T>, searchElement?: T['TValue'] | null, fromIndex?: number): number { + // Unions are special -- they do have a nullBitmap, but so can their children. + // If the searchElement is null, we don't know whether it came from the Union's + // bitmap or one of its childrens'. So we don't interrogate the Union's bitmap, + // since that will report the wrong index if a child has a null before the Union. + const compare = createElementComparator(searchElement); + for (let i = (fromIndex || 0) - 1, n = vector.length; ++i < n;) { + if (compare(vector.get(i))) { + return i; + } + } + return -1; +} + +IndexOfVisitor.prototype.visitNull = nullIndexOf; +IndexOfVisitor.prototype.visitBool = indexOfValue; +IndexOfVisitor.prototype.visitInt = indexOfValue; +IndexOfVisitor.prototype.visitInt8 = indexOfValue; +IndexOfVisitor.prototype.visitInt16 = indexOfValue; +IndexOfVisitor.prototype.visitInt32 = indexOfValue; +IndexOfVisitor.prototype.visitInt64 = indexOfValue; +IndexOfVisitor.prototype.visitUint8 = indexOfValue; +IndexOfVisitor.prototype.visitUint16 = indexOfValue; +IndexOfVisitor.prototype.visitUint32 = indexOfValue; +IndexOfVisitor.prototype.visitUint64 = indexOfValue; +IndexOfVisitor.prototype.visitFloat = indexOfValue; +IndexOfVisitor.prototype.visitFloat16 = indexOfValue; +IndexOfVisitor.prototype.visitFloat32 = indexOfValue; +IndexOfVisitor.prototype.visitFloat64 = indexOfValue; +IndexOfVisitor.prototype.visitUtf8 = indexOfValue; +IndexOfVisitor.prototype.visitBinary = indexOfValue; +IndexOfVisitor.prototype.visitFixedSizeBinary = indexOfValue; +IndexOfVisitor.prototype.visitDate = indexOfValue; +IndexOfVisitor.prototype.visitDateDay = indexOfValue; +IndexOfVisitor.prototype.visitDateMillisecond = indexOfValue; +IndexOfVisitor.prototype.visitTimestamp = indexOfValue; +IndexOfVisitor.prototype.visitTimestampSecond = indexOfValue; +IndexOfVisitor.prototype.visitTimestampMillisecond = indexOfValue; +IndexOfVisitor.prototype.visitTimestampMicrosecond = indexOfValue; +IndexOfVisitor.prototype.visitTimestampNanosecond = indexOfValue; +IndexOfVisitor.prototype.visitTime = indexOfValue; +IndexOfVisitor.prototype.visitTimeSecond = indexOfValue; +IndexOfVisitor.prototype.visitTimeMillisecond = indexOfValue; +IndexOfVisitor.prototype.visitTimeMicrosecond = indexOfValue; +IndexOfVisitor.prototype.visitTimeNanosecond = indexOfValue; +IndexOfVisitor.prototype.visitDecimal = indexOfValue; +IndexOfVisitor.prototype.visitList = indexOfValue; +IndexOfVisitor.prototype.visitStruct = indexOfValue; +IndexOfVisitor.prototype.visitUnion = indexOfValue; +IndexOfVisitor.prototype.visitDenseUnion = indexOfUnion; +IndexOfVisitor.prototype.visitSparseUnion = indexOfUnion; +IndexOfVisitor.prototype.visitDictionary = indexOfValue; +IndexOfVisitor.prototype.visitInterval = indexOfValue; +IndexOfVisitor.prototype.visitIntervalDayTime = indexOfValue; +IndexOfVisitor.prototype.visitIntervalYearMonth = indexOfValue; +IndexOfVisitor.prototype.visitFixedSizeList = indexOfValue; +IndexOfVisitor.prototype.visitMap = indexOfValue; + +/** @ignore */ +export const instance = new IndexOfVisitor(); diff --git a/src/arrow/js/src/visitor/iterator.ts b/src/arrow/js/src/visitor/iterator.ts new file mode 100644 index 000000000..4a8e6b5b6 --- /dev/null +++ b/src/arrow/js/src/visitor/iterator.ts @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import { Type } from '../enum'; +import { Visitor } from '../visitor'; +import { VectorType } from '../interfaces'; +import { BitIterator } from '../util/bit'; +import { instance as getVisitor } from './get'; +import { + DataType, Dictionary, + Bool, Null, Utf8, Binary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, + Float, Float16, Float32, Float64, + Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, + Date_, DateDay, DateMillisecond, + Interval, IntervalDayTime, IntervalYearMonth, + Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, + Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, + Union, DenseUnion, SparseUnion, +} from '../type'; + +/** @ignore */ +export interface IteratorVisitor extends Visitor { + visit<T extends VectorType>(node: T): IterableIterator<T['TValue'] | null>; + visitMany <T extends VectorType>(nodes: T[]): IterableIterator<T['TValue'] | null>[]; + getVisitFn<T extends Type>(node: T): (vector: VectorType<T>) => IterableIterator<VectorType<T>['TValue'] | null>; + getVisitFn<T extends DataType>(node: VectorType<T> | Data<T> | T): (vector: VectorType<T>) => IterableIterator<VectorType<T>['TValue'] | null>; + visitNull <T extends Null> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitBool <T extends Bool> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitInt <T extends Int> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitInt8 <T extends Int8> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitInt16 <T extends Int16> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitInt32 <T extends Int32> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitInt64 <T extends Int64> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitUint8 <T extends Uint8> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitUint16 <T extends Uint16> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitUint32 <T extends Uint32> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitUint64 <T extends Uint64> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitFloat <T extends Float> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitFloat16 <T extends Float16> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitFloat32 <T extends Float32> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitFloat64 <T extends Float64> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitUtf8 <T extends Utf8> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitBinary <T extends Binary> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitFixedSizeBinary <T extends FixedSizeBinary> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitDate <T extends Date_> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitDateDay <T extends DateDay> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitDateMillisecond <T extends DateMillisecond> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitTimestamp <T extends Timestamp> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitTimestampSecond <T extends TimestampSecond> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitTimestampMillisecond <T extends TimestampMillisecond> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitTimestampMicrosecond <T extends TimestampMicrosecond> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitTimestampNanosecond <T extends TimestampNanosecond> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitTime <T extends Time> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitTimeSecond <T extends TimeSecond> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitTimeMillisecond <T extends TimeMillisecond> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitTimeMicrosecond <T extends TimeMicrosecond> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitTimeNanosecond <T extends TimeNanosecond> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitDecimal <T extends Decimal> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitList <T extends List> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitStruct <T extends Struct> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitUnion <T extends Union> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitDenseUnion <T extends DenseUnion> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitSparseUnion <T extends SparseUnion> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitDictionary <T extends Dictionary> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitInterval <T extends Interval> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitIntervalDayTime <T extends IntervalDayTime> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitIntervalYearMonth <T extends IntervalYearMonth> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitFixedSizeList <T extends FixedSizeList> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; + visitMap <T extends Map_> (vector: VectorType<T>): IterableIterator<T['TValue'] | null>; +} + +/** @ignore */ +export class IteratorVisitor extends Visitor {} + +/** @ignore */ +function nullableIterator<T extends DataType>(vector: VectorType<T>): IterableIterator<T['TValue'] | null> { + const getFn = getVisitor.getVisitFn(vector); + return new BitIterator<T['TValue'] | null>( + vector.data.nullBitmap, vector.data.offset, vector.length, vector, + (vec: VectorType<T>, idx: number, nullByte: number, nullBit: number) => + ((nullByte & 1 << nullBit) !== 0) ? getFn(vec, idx) : null + ); +} + +/** @ignore */ +class VectorIterator<T extends DataType> implements IterableIterator<T['TValue'] | null> { + private index = 0; + + constructor( + private vector: VectorType<T>, + private getFn: (vector: VectorType<T>, index: number) => VectorType<T>['TValue'] + ) {} + + next(): IteratorResult<T['TValue'] | null> { + if (this.index < this.vector.length) { + return { + value: this.getFn(this.vector, this.index++) + }; + } + + return {done: true, value: null}; + } + + [Symbol.iterator]() { + return this; + } +} + +/** @ignore */ +function vectorIterator<T extends DataType>(vector: VectorType<T>): IterableIterator<T['TValue'] | null> { + + // If nullable, iterate manually + if (vector.nullCount > 0) { + return nullableIterator<T>(vector); + } + + const { type, typeId, length } = vector; + + // Fast case, defer to native iterators if possible + if (vector.stride === 1 && ( + (typeId === Type.Timestamp) || + (typeId === Type.Int && (type as Int).bitWidth !== 64) || + (typeId === Type.Time && (type as Time).bitWidth !== 64) || + (typeId === Type.Float && (type as Float).precision > 0 /* Precision.HALF */) + )) { + return vector.data.values.subarray(0, length)[Symbol.iterator](); + } + + // Otherwise, iterate manually + return new VectorIterator(vector, getVisitor.getVisitFn(vector)); +} + +IteratorVisitor.prototype.visitNull = vectorIterator; +IteratorVisitor.prototype.visitBool = vectorIterator; +IteratorVisitor.prototype.visitInt = vectorIterator; +IteratorVisitor.prototype.visitInt8 = vectorIterator; +IteratorVisitor.prototype.visitInt16 = vectorIterator; +IteratorVisitor.prototype.visitInt32 = vectorIterator; +IteratorVisitor.prototype.visitInt64 = vectorIterator; +IteratorVisitor.prototype.visitUint8 = vectorIterator; +IteratorVisitor.prototype.visitUint16 = vectorIterator; +IteratorVisitor.prototype.visitUint32 = vectorIterator; +IteratorVisitor.prototype.visitUint64 = vectorIterator; +IteratorVisitor.prototype.visitFloat = vectorIterator; +IteratorVisitor.prototype.visitFloat16 = vectorIterator; +IteratorVisitor.prototype.visitFloat32 = vectorIterator; +IteratorVisitor.prototype.visitFloat64 = vectorIterator; +IteratorVisitor.prototype.visitUtf8 = vectorIterator; +IteratorVisitor.prototype.visitBinary = vectorIterator; +IteratorVisitor.prototype.visitFixedSizeBinary = vectorIterator; +IteratorVisitor.prototype.visitDate = vectorIterator; +IteratorVisitor.prototype.visitDateDay = vectorIterator; +IteratorVisitor.prototype.visitDateMillisecond = vectorIterator; +IteratorVisitor.prototype.visitTimestamp = vectorIterator; +IteratorVisitor.prototype.visitTimestampSecond = vectorIterator; +IteratorVisitor.prototype.visitTimestampMillisecond = vectorIterator; +IteratorVisitor.prototype.visitTimestampMicrosecond = vectorIterator; +IteratorVisitor.prototype.visitTimestampNanosecond = vectorIterator; +IteratorVisitor.prototype.visitTime = vectorIterator; +IteratorVisitor.prototype.visitTimeSecond = vectorIterator; +IteratorVisitor.prototype.visitTimeMillisecond = vectorIterator; +IteratorVisitor.prototype.visitTimeMicrosecond = vectorIterator; +IteratorVisitor.prototype.visitTimeNanosecond = vectorIterator; +IteratorVisitor.prototype.visitDecimal = vectorIterator; +IteratorVisitor.prototype.visitList = vectorIterator; +IteratorVisitor.prototype.visitStruct = vectorIterator; +IteratorVisitor.prototype.visitUnion = vectorIterator; +IteratorVisitor.prototype.visitDenseUnion = vectorIterator; +IteratorVisitor.prototype.visitSparseUnion = vectorIterator; +IteratorVisitor.prototype.visitDictionary = vectorIterator; +IteratorVisitor.prototype.visitInterval = vectorIterator; +IteratorVisitor.prototype.visitIntervalDayTime = vectorIterator; +IteratorVisitor.prototype.visitIntervalYearMonth = vectorIterator; +IteratorVisitor.prototype.visitFixedSizeList = vectorIterator; +IteratorVisitor.prototype.visitMap = vectorIterator; + +/** @ignore */ +export const instance = new IteratorVisitor(); diff --git a/src/arrow/js/src/visitor/jsontypeassembler.ts b/src/arrow/js/src/visitor/jsontypeassembler.ts new file mode 100644 index 000000000..54f046f64 --- /dev/null +++ b/src/arrow/js/src/visitor/jsontypeassembler.ts @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import * as type from '../type'; +import { Visitor } from '../visitor'; +import { Type as ArrowType } from '../fb/Schema'; +import { Precision, DateUnit, TimeUnit, IntervalUnit, UnionMode } from '../enum'; + +/** @ignore */ +export interface JSONTypeAssembler extends Visitor { + visit<T extends type.DataType>(node: T): Record<string, unknown> | undefined; +} + +/** @ignore */ +export class JSONTypeAssembler extends Visitor { + public visit<T extends type.DataType>(node: T): Record<string, unknown> | undefined { + return node == null ? undefined : super.visit(node); + } + public visitNull<T extends type.Null>({ typeId }: T) { + return { 'name': ArrowType[typeId].toLowerCase() }; + } + public visitInt<T extends type.Int>({ typeId, bitWidth, isSigned }: T) { + return { 'name': ArrowType[typeId].toLowerCase(), 'bitWidth': bitWidth, 'isSigned': isSigned }; + } + public visitFloat<T extends type.Float>({ typeId, precision }: T) { + return { 'name': ArrowType[typeId].toLowerCase(), 'precision': Precision[precision] }; + } + public visitBinary<T extends type.Binary>({ typeId }: T) { + return { 'name': ArrowType[typeId].toLowerCase() }; + } + public visitBool<T extends type.Bool>({ typeId }: T) { + return { 'name': ArrowType[typeId].toLowerCase() }; + } + public visitUtf8<T extends type.Utf8>({ typeId }: T) { + return { 'name': ArrowType[typeId].toLowerCase() }; + } + public visitDecimal<T extends type.Decimal>({ typeId, scale, precision }: T) { + return { 'name': ArrowType[typeId].toLowerCase(), 'scale': scale, 'precision': precision }; + } + public visitDate<T extends type.Date_>({ typeId, unit }: T) { + return { 'name': ArrowType[typeId].toLowerCase(), 'unit': DateUnit[unit] }; + } + public visitTime<T extends type.Time>({ typeId, unit, bitWidth }: T) { + return { 'name': ArrowType[typeId].toLowerCase(), 'unit': TimeUnit[unit], bitWidth }; + } + public visitTimestamp<T extends type.Timestamp>({ typeId, timezone, unit }: T) { + return { 'name': ArrowType[typeId].toLowerCase(), 'unit': TimeUnit[unit], timezone }; + } + public visitInterval<T extends type.Interval>({ typeId, unit }: T) { + return { 'name': ArrowType[typeId].toLowerCase(), 'unit': IntervalUnit[unit] }; + } + public visitList<T extends type.List>({ typeId }: T) { + return { 'name': ArrowType[typeId].toLowerCase() }; + } + public visitStruct<T extends type.Struct>({ typeId }: T) { + return { 'name': ArrowType[typeId].toLowerCase() }; + } + public visitUnion<T extends type.Union>({ typeId, mode, typeIds }: T) { + return { + 'name': ArrowType[typeId].toLowerCase(), + 'mode': UnionMode[mode], + 'typeIds': [...typeIds] + }; + } + public visitDictionary<T extends type.Dictionary>(node: T) { + return this.visit(node.dictionary); + } + public visitFixedSizeBinary<T extends type.FixedSizeBinary>({ typeId, byteWidth }: T) { + return { 'name': ArrowType[typeId].toLowerCase(), 'byteWidth': byteWidth }; + } + public visitFixedSizeList<T extends type.FixedSizeList>({ typeId, listSize }: T) { + return { 'name': ArrowType[typeId].toLowerCase(), 'listSize': listSize }; + } + public visitMap<T extends type.Map_>({ typeId, keysSorted }: T) { + return { 'name': ArrowType[typeId].toLowerCase(), 'keysSorted': keysSorted }; + } +} diff --git a/src/arrow/js/src/visitor/jsonvectorassembler.ts b/src/arrow/js/src/visitor/jsonvectorassembler.ts new file mode 100644 index 000000000..f3c013344 --- /dev/null +++ b/src/arrow/js/src/visitor/jsonvectorassembler.ts @@ -0,0 +1,177 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { BN } from '../util/bn'; +import { Column } from '../column'; +import { Vector } from '../vector'; +import { Visitor } from '../visitor'; +import { BufferType } from '../enum'; +import { RecordBatch } from '../recordbatch'; +import { VectorType as V } from '../interfaces'; +import { UnionMode, DateUnit, TimeUnit } from '../enum'; +import { BitIterator, getBit, getBool } from '../util/bit'; +import { selectColumnChildrenArgs } from '../util/args'; +import { + DataType, + Float, Int, Date_, Interval, Time, Timestamp, Union, + Bool, Null, Utf8, Binary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, +} from '../type'; + +/** @ignore */ +export interface JSONVectorAssembler extends Visitor { + + visit <T extends Column> (node: T ): Record<string, unknown>; + visitMany <T extends Column> (cols: T[]): Record<string, unknown>[]; + getVisitFn<T extends DataType>(node: Column<T>): (column: Column<T>) => { name: string; count: number; VALIDITY: (0 | 1)[]; DATA?: any[]; OFFSET?: number[]; TYPE?: number[]; children?: any[] }; + + visitNull <T extends Null> (vector: V<T>): Record<string, never>; + visitBool <T extends Bool> (vector: V<T>): { DATA: boolean[] }; + visitInt <T extends Int> (vector: V<T>): { DATA: (number | string)[] }; + visitFloat <T extends Float> (vector: V<T>): { DATA: number[] }; + visitUtf8 <T extends Utf8> (vector: V<T>): { DATA: string[]; OFFSET: number[] }; + visitBinary <T extends Binary> (vector: V<T>): { DATA: string[]; OFFSET: number[] }; + visitFixedSizeBinary <T extends FixedSizeBinary> (vector: V<T>): { DATA: string[] }; + visitDate <T extends Date_> (vector: V<T>): { DATA: number[] }; + visitTimestamp <T extends Timestamp> (vector: V<T>): { DATA: string[] }; + visitTime <T extends Time> (vector: V<T>): { DATA: number[] }; + visitDecimal <T extends Decimal> (vector: V<T>): { DATA: string[] }; + visitList <T extends List> (vector: V<T>): { children: any[]; OFFSET: number[] }; + visitStruct <T extends Struct> (vector: V<T>): { children: any[] }; + visitUnion <T extends Union> (vector: V<T>): { children: any[]; TYPE: number[] }; + visitInterval <T extends Interval> (vector: V<T>): { DATA: number[] }; + visitFixedSizeList <T extends FixedSizeList> (vector: V<T>): { children: any[] }; + visitMap <T extends Map_> (vector: V<T>): { children: any[] }; +} + +/** @ignore */ +export class JSONVectorAssembler extends Visitor { + + /** @nocollapse */ + public static assemble<T extends Column | RecordBatch>(...args: (T | T[])[]) { + return new JSONVectorAssembler().visitMany(selectColumnChildrenArgs(RecordBatch, args)); + } + + public visit<T extends Column>(column: T) { + const { data, name, length } = column; + const { offset, nullCount, nullBitmap } = data; + const type = DataType.isDictionary(column.type) ? column.type.indices : column.type; + const buffers = Object.assign([], data.buffers, { [BufferType.VALIDITY]: undefined }); + return { + 'name': name, + 'count': length, + 'VALIDITY': DataType.isNull(type) ? undefined + : nullCount <= 0 ? Array.from({ length }, () => 1) + : [...new BitIterator(nullBitmap, offset, length, null, getBit)], + ...super.visit(Vector.new(data.clone(type, offset, length, 0, buffers))) + }; + } + public visitNull() { return {}; } + public visitBool<T extends Bool>({ values, offset, length }: V<T>) { + return { 'DATA': [...new BitIterator(values, offset, length, null, getBool)] }; + } + public visitInt<T extends Int>(vector: V<T>) { + return { + 'DATA': vector.type.bitWidth < 64 + ? [...vector.values] + : [...bigNumsToStrings(vector.values as (Int32Array | Uint32Array), 2)] + }; + } + public visitFloat<T extends Float>(vector: V<T>) { + return { 'DATA': [...vector.values] }; + } + public visitUtf8<T extends Utf8>(vector: V<T>) { + return { 'DATA': [...vector], 'OFFSET': [...vector.valueOffsets] }; + } + public visitBinary<T extends Binary>(vector: V<T>) { + return { 'DATA': [...binaryToString(vector)], OFFSET: [...vector.valueOffsets] }; + } + public visitFixedSizeBinary<T extends FixedSizeBinary>(vector: V<T>) { + return { 'DATA': [...binaryToString(vector)] }; + } + public visitDate<T extends Date_>(vector: V<T>) { + return { + 'DATA': vector.type.unit === DateUnit.DAY + ? [...vector.values] + : [...bigNumsToStrings(vector.values, 2)] + }; + } + public visitTimestamp<T extends Timestamp>(vector: V<T>) { + return { 'DATA': [...bigNumsToStrings(vector.values, 2)] }; + } + public visitTime<T extends Time>(vector: V<T>) { + return { + 'DATA': vector.type.unit < TimeUnit.MICROSECOND + ? [...vector.values] + : [...bigNumsToStrings(vector.values, 2)] + }; + } + public visitDecimal<T extends Decimal>(vector: V<T>) { + return { 'DATA': [...bigNumsToStrings(vector.values, 4)] }; + } + public visitList<T extends List>(vector: V<T>) { + return { + 'OFFSET': [...vector.valueOffsets], + 'children': vector.type.children.map((f, i) => + this.visit(new Column(f, [vector.getChildAt(i)!]))) + }; + } + public visitStruct<T extends Struct>(vector: V<T>) { + return { + 'children': vector.type.children.map((f, i) => + this.visit(new Column(f, [vector.getChildAt(i)!]))) + }; + } + public visitUnion<T extends Union>(vector: V<T>) { + return { + 'TYPE': [...vector.typeIds], + 'OFFSET': vector.type.mode === UnionMode.Dense ? [...vector.valueOffsets] : undefined, + 'children': vector.type.children.map((f, i) => this.visit(new Column(f, [vector.getChildAt(i)!]))) + }; + } + public visitInterval<T extends Interval>(vector: V<T>) { + return { 'DATA': [...vector.values] }; + } + public visitFixedSizeList<T extends FixedSizeList>(vector: V<T>) { + return { + 'children': vector.type.children.map((f, i) => + this.visit(new Column(f, [vector.getChildAt(i)!]))) + }; + } + public visitMap<T extends Map_>(vector: V<T>) { + return { + 'OFFSET': [...vector.valueOffsets], + 'children': vector.type.children.map((f, i) => + this.visit(new Column(f, [vector.getChildAt(i)!]))) + }; + } +} + +/** @ignore */ +function* binaryToString(vector: Vector<Binary> | Vector<FixedSizeBinary>) { + for (const octets of vector as Iterable<Uint8Array>) { + yield octets.reduce((str, byte) => { + return `${str}${('0' + (byte & 0xFF).toString(16)).slice(-2)}`; + }, '').toUpperCase(); + } +} + +/** @ignore */ +function* bigNumsToStrings(values: Uint32Array | Int32Array, stride: number) { + for (let i = -1, n = values.length / stride; ++i < n;) { + yield `${BN.new(values.subarray((i + 0) * stride, (i + 1) * stride), false)}`; + } +} diff --git a/src/arrow/js/src/visitor/set.ts b/src/arrow/js/src/visitor/set.ts new file mode 100644 index 000000000..77985e5be --- /dev/null +++ b/src/arrow/js/src/visitor/set.ts @@ -0,0 +1,354 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import { Field } from '../schema'; +import { Vector } from '../vector'; +import { Visitor } from '../visitor'; +import { encodeUtf8 } from '../util/utf8'; +import { VectorType } from '../interfaces'; +import { float64ToUint16 } from '../util/math'; +import { toArrayBufferView } from '../util/buffer'; +import { Type, UnionMode, Precision, DateUnit, TimeUnit, IntervalUnit } from '../enum'; +import { + DataType, Dictionary, + Bool, Null, Utf8, Binary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, + Float, Float16, Float32, Float64, + Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, + Date_, DateDay, DateMillisecond, + Interval, IntervalDayTime, IntervalYearMonth, + Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, + Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, + Union, DenseUnion, SparseUnion, +} from '../type'; + +/** @ignore */ +export interface SetVisitor extends Visitor { + visit<T extends VectorType>(node: T, index: number, value: T['TValue']): void; + visitMany<T extends VectorType>(nodes: T[], indices: number[], values: T['TValue'][]): void[]; + getVisitFn<T extends Type>(node: T): (vector: VectorType<T>, index: number, value: VectorType<T>['TValue']) => void; + getVisitFn<T extends DataType>(node: VectorType<T> | Data<T> | T): (vector: VectorType<T>, index: number, value: VectorType<T>['TValue']) => void; + visitNull <T extends Null> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitBool <T extends Bool> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitInt <T extends Int> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitInt8 <T extends Int8> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitInt16 <T extends Int16> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitInt32 <T extends Int32> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitInt64 <T extends Int64> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitUint8 <T extends Uint8> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitUint16 <T extends Uint16> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitUint32 <T extends Uint32> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitUint64 <T extends Uint64> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitFloat <T extends Float> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitFloat16 <T extends Float16> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitFloat32 <T extends Float32> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitFloat64 <T extends Float64> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitUtf8 <T extends Utf8> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitBinary <T extends Binary> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitFixedSizeBinary <T extends FixedSizeBinary> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitDate <T extends Date_> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitDateDay <T extends DateDay> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitDateMillisecond <T extends DateMillisecond> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitTimestamp <T extends Timestamp> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitTimestampSecond <T extends TimestampSecond> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitTimestampMillisecond <T extends TimestampMillisecond>(vector: VectorType<T>, index: number, value: T['TValue']): void; + visitTimestampMicrosecond <T extends TimestampMicrosecond>(vector: VectorType<T>, index: number, value: T['TValue']): void; + visitTimestampNanosecond <T extends TimestampNanosecond> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitTime <T extends Time> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitTimeSecond <T extends TimeSecond> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitTimeMillisecond <T extends TimeMillisecond> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitTimeMicrosecond <T extends TimeMicrosecond> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitTimeNanosecond <T extends TimeNanosecond> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitDecimal <T extends Decimal> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitList <T extends List> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitStruct <T extends Struct> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitUnion <T extends Union> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitDenseUnion <T extends DenseUnion> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitSparseUnion <T extends SparseUnion> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitDictionary <T extends Dictionary> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitInterval <T extends Interval> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitIntervalDayTime <T extends IntervalDayTime> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitIntervalYearMonth <T extends IntervalYearMonth> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitFixedSizeList <T extends FixedSizeList> (vector: VectorType<T>, index: number, value: T['TValue']): void; + visitMap <T extends Map_> (vector: VectorType<T>, index: number, value: T['TValue']): void; +} + +/** @ignore */ +export class SetVisitor extends Visitor {} + +/** @ignore */ +const setEpochMsToDays = (data: Int32Array, index: number, epochMs: number) => { data[index] = (epochMs / 86400000) | 0; }; +/** @ignore */ +const setEpochMsToMillisecondsLong = (data: Int32Array, index: number, epochMs: number) => { + data[index] = (epochMs % 4294967296) | 0; + data[index + 1] = (epochMs / 4294967296) | 0; +}; +/** @ignore */ +const setEpochMsToMicrosecondsLong = (data: Int32Array, index: number, epochMs: number) => { + data[index] = ((epochMs * 1000) % 4294967296) | 0; + data[index + 1] = ((epochMs * 1000) / 4294967296) | 0; +}; +/** @ignore */ +const setEpochMsToNanosecondsLong = (data: Int32Array, index: number, epochMs: number) => { + data[index] = ((epochMs * 1000000) % 4294967296) | 0; + data[index + 1] = ((epochMs * 1000000) / 4294967296) | 0; +}; + +/** @ignore */ +const setVariableWidthBytes = (values: Uint8Array, valueOffsets: Int32Array, index: number, value: Uint8Array) => { + const { [index]: x, [index + 1]: y } = valueOffsets; + if (x != null && y != null) { + values.set(value.subarray(0, y - x), x); + } +}; + +/** @ignore */ +const setBool = <T extends Bool>({ offset, values }: VectorType<T>, index: number, val: boolean) => { + const idx = offset + index; + val ? (values[idx >> 3] |= (1 << (idx % 8))) // true + : (values[idx >> 3] &= ~(1 << (idx % 8))); // false + +}; + +/** @ignore */ type Numeric1X = Int8 | Int16 | Int32 | Uint8 | Uint16 | Uint32 | Float32 | Float64; +/** @ignore */ type Numeric2X = Int64 | Uint64; + +/** @ignore */ +const setDateDay = <T extends DateDay> ({ values }: VectorType<T>, index: number, value: T['TValue']): void => { setEpochMsToDays(values, index, value.valueOf()); }; +/** @ignore */ +const setDateMillisecond = <T extends DateMillisecond>({ values }: VectorType<T>, index: number, value: T['TValue']): void => { setEpochMsToMillisecondsLong(values, index * 2, value.valueOf()); }; +/** @ignore */ +const setNumeric = <T extends Numeric1X> ({ stride, values }: VectorType<T>, index: number, value: T['TValue']): void => { values[stride * index] = value; }; +/** @ignore */ +const setFloat16 = <T extends Float16> ({ stride, values }: VectorType<T>, index: number, value: T['TValue']): void => { values[stride * index] = float64ToUint16(value); }; +/** @ignore */ +const setNumericX2 = <T extends Numeric2X> (vector: VectorType<T>, index: number, value: T['TValue']): void => { + switch (typeof value) { + case 'bigint': vector.values64[index] = value; break; + case 'number': vector.values[index * vector.stride] = value; break; + default: { + const val = value as T['TArray']; + const { stride, ArrayType } = vector; + const long = toArrayBufferView<T['TArray']>(ArrayType, val); + vector.values.set(long.subarray(0, stride), stride * index); + } + } +}; +/** @ignore */ +const setFixedSizeBinary = <T extends FixedSizeBinary>({ stride, values }: VectorType<T>, index: number, value: T['TValue']): void => { values.set(value.subarray(0, stride), stride * index); }; + +/** @ignore */ +const setBinary = <T extends Binary>({ values, valueOffsets }: VectorType<T>, index: number, value: T['TValue']) => setVariableWidthBytes(values, valueOffsets, index, value); +/** @ignore */ +const setUtf8 = <T extends Utf8>({ values, valueOffsets }: VectorType<T>, index: number, value: T['TValue']) => { + setVariableWidthBytes(values, valueOffsets, index, encodeUtf8(value)); +}; + +/* istanbul ignore next */ +/** @ignore */ +const setInt = <T extends Int>(vector: VectorType<T>, index: number, value: T['TValue']): void => { + vector.type.bitWidth < 64 + ? setNumeric(vector as VectorType<Numeric1X>, index, value as Numeric1X['TValue']) + : setNumericX2(vector as VectorType<Numeric2X>, index, value as Numeric2X['TValue']); +}; + +/* istanbul ignore next */ +/** @ignore */ +const setFloat = <T extends Float>(vector: VectorType<T>, index: number, value: T['TValue']): void => { + vector.type.precision !== Precision.HALF + ? setNumeric(vector as VectorType<Numeric1X>, index, value) + : setFloat16(vector as VectorType<Float16>, index, value); +}; + +/* istanbul ignore next */ +const setDate = <T extends Date_> (vector: VectorType<T>, index: number, value: T['TValue']): void => { + vector.type.unit === DateUnit.DAY + ? setDateDay(vector as VectorType<DateDay>, index, value) + : setDateMillisecond(vector as VectorType<DateMillisecond>, index, value); +}; + +/** @ignore */ +const setTimestampSecond = <T extends TimestampSecond> ({ values }: VectorType<T>, index: number, value: T['TValue']): void => setEpochMsToMillisecondsLong(values, index * 2, value / 1000); +/** @ignore */ +const setTimestampMillisecond = <T extends TimestampMillisecond>({ values }: VectorType<T>, index: number, value: T['TValue']): void => setEpochMsToMillisecondsLong(values, index * 2, value); +/** @ignore */ +const setTimestampMicrosecond = <T extends TimestampMicrosecond>({ values }: VectorType<T>, index: number, value: T['TValue']): void => setEpochMsToMicrosecondsLong(values, index * 2, value); +/** @ignore */ +const setTimestampNanosecond = <T extends TimestampNanosecond> ({ values }: VectorType<T>, index: number, value: T['TValue']): void => setEpochMsToNanosecondsLong(values, index * 2, value); +/* istanbul ignore next */ +/** @ignore */ +const setTimestamp = <T extends Timestamp>(vector: VectorType<T>, index: number, value: T['TValue']): void => { + switch (vector.type.unit) { + case TimeUnit.SECOND: return setTimestampSecond(vector as VectorType<TimestampSecond>, index, value); + case TimeUnit.MILLISECOND: return setTimestampMillisecond(vector as VectorType<TimestampMillisecond>, index, value); + case TimeUnit.MICROSECOND: return setTimestampMicrosecond(vector as VectorType<TimestampMicrosecond>, index, value); + case TimeUnit.NANOSECOND: return setTimestampNanosecond(vector as VectorType<TimestampNanosecond>, index, value); + } +}; + +/** @ignore */ +const setTimeSecond = <T extends TimeSecond> ({ values, stride }: VectorType<T>, index: number, value: T['TValue']): void => { values[stride * index] = value; }; +/** @ignore */ +const setTimeMillisecond = <T extends TimeMillisecond>({ values, stride }: VectorType<T>, index: number, value: T['TValue']): void => { values[stride * index] = value; }; +/** @ignore */ +const setTimeMicrosecond = <T extends TimeMicrosecond>({ values }: VectorType<T>, index: number, value: T['TValue']): void => { values.set(value.subarray(0, 2), 2 * index); }; +/** @ignore */ +const setTimeNanosecond = <T extends TimeNanosecond> ({ values }: VectorType<T>, index: number, value: T['TValue']): void => { values.set(value.subarray(0, 2), 2 * index); }; +/* istanbul ignore next */ +/** @ignore */ +const setTime = <T extends Time>(vector: VectorType<T>, index: number, value: T['TValue']): void => { + switch (vector.type.unit) { + case TimeUnit.SECOND: return setTimeSecond(vector as VectorType<TimeSecond>, index, value as TimeSecond['TValue']); + case TimeUnit.MILLISECOND: return setTimeMillisecond(vector as VectorType<TimeMillisecond>, index, value as TimeMillisecond['TValue']); + case TimeUnit.MICROSECOND: return setTimeMicrosecond(vector as VectorType<TimeMicrosecond>, index, value as TimeMicrosecond['TValue']); + case TimeUnit.NANOSECOND: return setTimeNanosecond(vector as VectorType<TimeNanosecond>, index, value as TimeNanosecond['TValue']); + } +}; + +/** @ignore */ +const setDecimal = <T extends Decimal>({ values }: VectorType<T>, index: number, value: T['TValue']): void => { values.set(value.subarray(0, 4), 4 * index); }; + +/** @ignore */ +const setList = <T extends List>(vector: VectorType<T>, index: number, value: T['TValue']): void => { + const values = vector.getChildAt(0)!, valueOffsets = vector.valueOffsets; + for (let idx = -1, itr = valueOffsets[index], end = valueOffsets[index + 1]; itr < end;) { + values.set(itr++, value.get(++idx)); + } +}; + +/** @ignore */ +const setMap = <T extends Map_>(vector: VectorType<T>, index: number, value: T['TValue']) => { + const values = vector.getChildAt(0)!, valueOffsets = vector.valueOffsets; + const entries = value instanceof Map ? [...value] : Object.entries(value); + for (let idx = -1, itr = valueOffsets[index], end = valueOffsets[index + 1]; itr < end;) { + values.set(itr++, entries[++idx]); + } +}; + +/** @ignore */ const _setStructArrayValue = (o: number, v: any[]) => (c: Vector | null, _: Field, i: number) => c?.set(o, v[i]); +/** @ignore */ const _setStructVectorValue = (o: number, v: Vector) => (c: Vector | null, _: Field, i: number) => c?.set(o, v.get(i)); +/** @ignore */ const _setStructMapValue = (o: number, v: Map<string, any>) => (c: Vector | null, f: Field, _: number) => c?.set(o, v.get(f.name)); +/** @ignore */ const _setStructObjectValue = (o: number, v: { [key: string]: any }) => (c: Vector | null, f: Field, _: number) => c?.set(o, v[f.name]); +/** @ignore */ +const setStruct = <T extends Struct>(vector: VectorType<T>, index: number, value: T['TValue']) => { + + const setValue = value instanceof Map ? _setStructMapValue(index, value) : + value instanceof Vector ? _setStructVectorValue(index, value) : + Array.isArray(value) ? _setStructArrayValue(index, value) : + _setStructObjectValue(index, value) ; + + vector.type.children.forEach((f: Field, i: number) => setValue(vector.getChildAt(i), f, i)); +}; + +/* istanbul ignore next */ +/** @ignore */ +const setUnion = < + V extends VectorType<Union> | VectorType<DenseUnion> | VectorType<SparseUnion> +>(vector: V, index: number, value: V['TValue']) => { + vector.type.mode === UnionMode.Dense ? + setDenseUnion(vector as VectorType<DenseUnion>, index, value) : + setSparseUnion(vector as VectorType<SparseUnion>, index, value); +}; + +/** @ignore */ +const setDenseUnion = <T extends DenseUnion>(vector: VectorType<T>, index: number, value: T['TValue']): void => { + const childIndex = vector.typeIdToChildIndex[vector.typeIds[index]]; + const child = vector.getChildAt(childIndex); + child && child.set(vector.valueOffsets[index], value); +}; + +/** @ignore */ +const setSparseUnion = <T extends SparseUnion>(vector: VectorType<T>, index: number, value: T['TValue']): void => { + const childIndex = vector.typeIdToChildIndex[vector.typeIds[index]]; + const child = vector.getChildAt(childIndex); + child && child.set(index, value); +}; + +/** @ignore */ +const setDictionary = <T extends Dictionary>(vector: VectorType<T>, index: number, value: T['TValue']): void => { + const key = vector.getKey(index); + if (key !== null) { + vector.setValue(key, value); + } +}; + +/* istanbul ignore next */ +/** @ignore */ +const setIntervalValue = <T extends Interval>(vector: VectorType<T>, index: number, value: T['TValue']): void => { + (vector.type.unit === IntervalUnit.DAY_TIME) + ? setIntervalDayTime(vector as VectorType<IntervalDayTime>, index, value) + : setIntervalYearMonth(vector as VectorType<IntervalYearMonth>, index, value); +}; + +/** @ignore */ +const setIntervalDayTime = <T extends IntervalDayTime>({ values }: VectorType<T>, index: number, value: T['TValue']): void => { values.set(value.subarray(0, 2), 2 * index); }; +/** @ignore */ +const setIntervalYearMonth = <T extends IntervalYearMonth>({ values }: VectorType<T>, index: number, value: T['TValue']): void => { values[index] = (value[0] * 12) + (value[1] % 12); }; + +/** @ignore */ +const setFixedSizeList = <T extends FixedSizeList>(vector: VectorType<T>, index: number, value: T['TValue']): void => { + const child = vector.getChildAt(0)!, { stride } = vector; + for (let idx = -1, offset = index * stride; ++idx < stride;) { + child.set(offset + idx, value.get(idx)); + } +}; + +SetVisitor.prototype.visitBool = setBool; +SetVisitor.prototype.visitInt = setInt; +SetVisitor.prototype.visitInt8 = setNumeric; +SetVisitor.prototype.visitInt16 = setNumeric; +SetVisitor.prototype.visitInt32 = setNumeric; +SetVisitor.prototype.visitInt64 = setNumericX2; +SetVisitor.prototype.visitUint8 = setNumeric; +SetVisitor.prototype.visitUint16 = setNumeric; +SetVisitor.prototype.visitUint32 = setNumeric; +SetVisitor.prototype.visitUint64 = setNumericX2; +SetVisitor.prototype.visitFloat = setFloat; +SetVisitor.prototype.visitFloat16 = setFloat16; +SetVisitor.prototype.visitFloat32 = setNumeric; +SetVisitor.prototype.visitFloat64 = setNumeric; +SetVisitor.prototype.visitUtf8 = setUtf8; +SetVisitor.prototype.visitBinary = setBinary; +SetVisitor.prototype.visitFixedSizeBinary = setFixedSizeBinary; +SetVisitor.prototype.visitDate = setDate; +SetVisitor.prototype.visitDateDay = setDateDay; +SetVisitor.prototype.visitDateMillisecond = setDateMillisecond; +SetVisitor.prototype.visitTimestamp = setTimestamp; +SetVisitor.prototype.visitTimestampSecond = setTimestampSecond; +SetVisitor.prototype.visitTimestampMillisecond = setTimestampMillisecond; +SetVisitor.prototype.visitTimestampMicrosecond = setTimestampMicrosecond; +SetVisitor.prototype.visitTimestampNanosecond = setTimestampNanosecond; +SetVisitor.prototype.visitTime = setTime; +SetVisitor.prototype.visitTimeSecond = setTimeSecond; +SetVisitor.prototype.visitTimeMillisecond = setTimeMillisecond; +SetVisitor.prototype.visitTimeMicrosecond = setTimeMicrosecond; +SetVisitor.prototype.visitTimeNanosecond = setTimeNanosecond; +SetVisitor.prototype.visitDecimal = setDecimal; +SetVisitor.prototype.visitList = setList; +SetVisitor.prototype.visitStruct = setStruct; +SetVisitor.prototype.visitUnion = setUnion; +SetVisitor.prototype.visitDenseUnion = setDenseUnion; +SetVisitor.prototype.visitSparseUnion = setSparseUnion; +SetVisitor.prototype.visitDictionary = setDictionary; +SetVisitor.prototype.visitInterval = setIntervalValue; +SetVisitor.prototype.visitIntervalDayTime = setIntervalDayTime; +SetVisitor.prototype.visitIntervalYearMonth = setIntervalYearMonth; +SetVisitor.prototype.visitFixedSizeList = setFixedSizeList; +SetVisitor.prototype.visitMap = setMap; + +/** @ignore */ +export const instance = new SetVisitor(); diff --git a/src/arrow/js/src/visitor/toarray.ts b/src/arrow/js/src/visitor/toarray.ts new file mode 100644 index 000000000..395e9943c --- /dev/null +++ b/src/arrow/js/src/visitor/toarray.ts @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import { Type } from '../enum'; +import { Visitor } from '../visitor'; +import { VectorType } from '../interfaces'; +import { instance as iteratorVisitor } from './iterator'; +import { + DataType, Dictionary, + Bool, Null, Utf8, Binary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, + Float, Float16, Float32, Float64, + Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, + Date_, DateDay, DateMillisecond, + Interval, IntervalDayTime, IntervalYearMonth, + Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, + Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, + Union, DenseUnion, SparseUnion, +} from '../type'; + +/** @ignore */ +export interface ToArrayVisitor extends Visitor { + visit<T extends VectorType>(node: T): T['TArray']; + visitMany<T extends VectorType>(nodes: T[]): T['TArray'][]; + getVisitFn<T extends Type>(node: T): (vector: VectorType<T>) => VectorType<T>['TArray']; + getVisitFn<T extends DataType>(node: VectorType<T> | Data<T> | T): (vector: VectorType<T>) => VectorType<T>['TArray']; + visitNull <T extends Null> (vector: VectorType<T>): VectorType<T>['TArray']; + visitBool <T extends Bool> (vector: VectorType<T>): VectorType<T>['TArray']; + visitInt <T extends Int> (vector: VectorType<T>): VectorType<T>['TArray']; + visitInt8 <T extends Int8> (vector: VectorType<T>): VectorType<T>['TArray']; + visitInt16 <T extends Int16> (vector: VectorType<T>): VectorType<T>['TArray']; + visitInt32 <T extends Int32> (vector: VectorType<T>): VectorType<T>['TArray']; + visitInt64 <T extends Int64> (vector: VectorType<T>): VectorType<T>['TArray']; + visitUint8 <T extends Uint8> (vector: VectorType<T>): VectorType<T>['TArray']; + visitUint16 <T extends Uint16> (vector: VectorType<T>): VectorType<T>['TArray']; + visitUint32 <T extends Uint32> (vector: VectorType<T>): VectorType<T>['TArray']; + visitUint64 <T extends Uint64> (vector: VectorType<T>): VectorType<T>['TArray']; + visitFloat <T extends Float> (vector: VectorType<T>): VectorType<T>['TArray']; + visitFloat16 <T extends Float16> (vector: VectorType<T>): VectorType<T>['TArray']; + visitFloat32 <T extends Float32> (vector: VectorType<T>): VectorType<T>['TArray']; + visitFloat64 <T extends Float64> (vector: VectorType<T>): VectorType<T>['TArray']; + visitUtf8 <T extends Utf8> (vector: VectorType<T>): VectorType<T>['TArray']; + visitBinary <T extends Binary> (vector: VectorType<T>): VectorType<T>['TArray']; + visitFixedSizeBinary <T extends FixedSizeBinary> (vector: VectorType<T>): VectorType<T>['TArray']; + visitDate <T extends Date_> (vector: VectorType<T>): VectorType<T>['TArray']; + visitDateDay <T extends DateDay> (vector: VectorType<T>): VectorType<T>['TArray']; + visitDateMillisecond <T extends DateMillisecond> (vector: VectorType<T>): VectorType<T>['TArray']; + visitTimestamp <T extends Timestamp> (vector: VectorType<T>): VectorType<T>['TArray']; + visitTimestampSecond <T extends TimestampSecond> (vector: VectorType<T>): VectorType<T>['TArray']; + visitTimestampMillisecond <T extends TimestampMillisecond>(vector: VectorType<T>): VectorType<T>['TArray']; + visitTimestampMicrosecond <T extends TimestampMicrosecond>(vector: VectorType<T>): VectorType<T>['TArray']; + visitTimestampNanosecond <T extends TimestampNanosecond> (vector: VectorType<T>): VectorType<T>['TArray']; + visitTime <T extends Time> (vector: VectorType<T>): VectorType<T>['TArray']; + visitTimeSecond <T extends TimeSecond> (vector: VectorType<T>): VectorType<T>['TArray']; + visitTimeMillisecond <T extends TimeMillisecond> (vector: VectorType<T>): VectorType<T>['TArray']; + visitTimeMicrosecond <T extends TimeMicrosecond> (vector: VectorType<T>): VectorType<T>['TArray']; + visitTimeNanosecond <T extends TimeNanosecond> (vector: VectorType<T>): VectorType<T>['TArray']; + visitDecimal <T extends Decimal> (vector: VectorType<T>): VectorType<T>['TArray']; + visitList <R extends DataType, T extends List<R>> (vector: VectorType<T>): VectorType<T>['TArray']; + visitStruct <T extends Struct> (vector: VectorType<T>): VectorType<T>['TArray']; + visitUnion <T extends Union> (vector: VectorType<T>): VectorType<T>['TArray']; + visitDenseUnion <T extends DenseUnion> (vector: VectorType<T>): VectorType<T>['TArray']; + visitSparseUnion <T extends SparseUnion> (vector: VectorType<T>): VectorType<T>['TArray']; + visitDictionary <R extends DataType, T extends Dictionary<R>> (vector: VectorType<T>): VectorType<T>['TArray']; + visitInterval <T extends Interval> (vector: VectorType<T>): VectorType<T>['TArray']; + visitIntervalDayTime <T extends IntervalDayTime> (vector: VectorType<T>): VectorType<T>['TArray']; + visitIntervalYearMonth <T extends IntervalYearMonth> (vector: VectorType<T>): VectorType<T>['TArray']; + visitFixedSizeList <R extends DataType, T extends FixedSizeList<R>> (vector: VectorType<T>): VectorType<T>['TArray']; + visitMap <T extends Map_> (vector: VectorType<T>): VectorType<T>['TArray']; +} + +/** @ignore */ +export class ToArrayVisitor extends Visitor {} + +/** @ignore */ +function arrayOfVector<T extends DataType>(vector: VectorType<T>): T['TArray'] { + + const { type, length, stride } = vector; + + // Fast case, return subarray if possible + switch (type.typeId) { + case Type.Int: + case Type.Float: case Type.Decimal: + case Type.Time: case Type.Timestamp: + return vector.data.values.subarray(0, length * stride); + } + + // Otherwise if not primitive, slow copy + return [...iteratorVisitor.visit(vector)] as T['TArray']; +} + +ToArrayVisitor.prototype.visitNull = arrayOfVector; +ToArrayVisitor.prototype.visitBool = arrayOfVector; +ToArrayVisitor.prototype.visitInt = arrayOfVector; +ToArrayVisitor.prototype.visitInt8 = arrayOfVector; +ToArrayVisitor.prototype.visitInt16 = arrayOfVector; +ToArrayVisitor.prototype.visitInt32 = arrayOfVector; +ToArrayVisitor.prototype.visitInt64 = arrayOfVector; +ToArrayVisitor.prototype.visitUint8 = arrayOfVector; +ToArrayVisitor.prototype.visitUint16 = arrayOfVector; +ToArrayVisitor.prototype.visitUint32 = arrayOfVector; +ToArrayVisitor.prototype.visitUint64 = arrayOfVector; +ToArrayVisitor.prototype.visitFloat = arrayOfVector; +ToArrayVisitor.prototype.visitFloat16 = arrayOfVector; +ToArrayVisitor.prototype.visitFloat32 = arrayOfVector; +ToArrayVisitor.prototype.visitFloat64 = arrayOfVector; +ToArrayVisitor.prototype.visitUtf8 = arrayOfVector; +ToArrayVisitor.prototype.visitBinary = arrayOfVector; +ToArrayVisitor.prototype.visitFixedSizeBinary = arrayOfVector; +ToArrayVisitor.prototype.visitDate = arrayOfVector; +ToArrayVisitor.prototype.visitDateDay = arrayOfVector; +ToArrayVisitor.prototype.visitDateMillisecond = arrayOfVector; +ToArrayVisitor.prototype.visitTimestamp = arrayOfVector; +ToArrayVisitor.prototype.visitTimestampSecond = arrayOfVector; +ToArrayVisitor.prototype.visitTimestampMillisecond = arrayOfVector; +ToArrayVisitor.prototype.visitTimestampMicrosecond = arrayOfVector; +ToArrayVisitor.prototype.visitTimestampNanosecond = arrayOfVector; +ToArrayVisitor.prototype.visitTime = arrayOfVector; +ToArrayVisitor.prototype.visitTimeSecond = arrayOfVector; +ToArrayVisitor.prototype.visitTimeMillisecond = arrayOfVector; +ToArrayVisitor.prototype.visitTimeMicrosecond = arrayOfVector; +ToArrayVisitor.prototype.visitTimeNanosecond = arrayOfVector; +ToArrayVisitor.prototype.visitDecimal = arrayOfVector; +ToArrayVisitor.prototype.visitList = arrayOfVector; +ToArrayVisitor.prototype.visitStruct = arrayOfVector; +ToArrayVisitor.prototype.visitUnion = arrayOfVector; +ToArrayVisitor.prototype.visitDenseUnion = arrayOfVector; +ToArrayVisitor.prototype.visitSparseUnion = arrayOfVector; +ToArrayVisitor.prototype.visitDictionary = arrayOfVector; +ToArrayVisitor.prototype.visitInterval = arrayOfVector; +ToArrayVisitor.prototype.visitIntervalDayTime = arrayOfVector; +ToArrayVisitor.prototype.visitIntervalYearMonth = arrayOfVector; +ToArrayVisitor.prototype.visitFixedSizeList = arrayOfVector; +ToArrayVisitor.prototype.visitMap = arrayOfVector; + +/** @ignore */ +export const instance = new ToArrayVisitor(); diff --git a/src/arrow/js/src/visitor/typeassembler.ts b/src/arrow/js/src/visitor/typeassembler.ts new file mode 100644 index 000000000..4cd65d926 --- /dev/null +++ b/src/arrow/js/src/visitor/typeassembler.ts @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { flatbuffers } from 'flatbuffers'; +import Long = flatbuffers.Long; +import Builder = flatbuffers.Builder; + +import * as type from '../type'; +import { Visitor } from '../visitor'; + +import { + Null, + Int, + FloatingPoint, + Binary, + Bool, + Utf8, + Decimal, + Date, + Time, + Timestamp, + Interval, + List, + Struct_ as Struct, + Union, + DictionaryEncoding, + FixedSizeBinary, + FixedSizeList, + Map as Map_, +} from '../fb/Schema'; + +/** @ignore */ +export interface TypeAssembler extends Visitor { + visit<T extends type.DataType>(node: T, builder: Builder): number | undefined; +} + +/** @ignore */ +export class TypeAssembler extends Visitor { + public visit<T extends type.DataType>(node: T, builder: Builder): number | undefined { + return (node == null || builder == null) ? undefined : super.visit(node, builder); + } + public visitNull<T extends type.Null>(_node: T, b: Builder) { + Null.startNull(b); + return Null.endNull(b); + } + public visitInt<T extends type.Int>(node: T, b: Builder) { + Int.startInt(b); + Int.addBitWidth(b, node.bitWidth); + Int.addIsSigned(b, node.isSigned); + return Int.endInt(b); + } + public visitFloat<T extends type.Float>(node: T, b: Builder) { + FloatingPoint.startFloatingPoint(b); + FloatingPoint.addPrecision(b, node.precision); + return FloatingPoint.endFloatingPoint(b); + } + public visitBinary<T extends type.Binary>(_node: T, b: Builder) { + Binary.startBinary(b); + return Binary.endBinary(b); + } + public visitBool<T extends type.Bool>(_node: T, b: Builder) { + Bool.startBool(b); + return Bool.endBool(b); + } + public visitUtf8<T extends type.Utf8>(_node: T, b: Builder) { + Utf8.startUtf8(b); + return Utf8.endUtf8(b); + } + public visitDecimal<T extends type.Decimal>(node: T, b: Builder) { + Decimal.startDecimal(b); + Decimal.addScale(b, node.scale); + Decimal.addPrecision(b, node.precision); + return Decimal.endDecimal(b); + } + public visitDate<T extends type.Date_>(node: T, b: Builder) { + Date.startDate(b); + Date.addUnit(b, node.unit); + return Date.endDate(b); + } + public visitTime<T extends type.Time>(node: T, b: Builder) { + Time.startTime(b); + Time.addUnit(b, node.unit); + Time.addBitWidth(b, node.bitWidth); + return Time.endTime(b); + } + public visitTimestamp<T extends type.Timestamp>(node: T, b: Builder) { + const timezone = (node.timezone && b.createString(node.timezone)) || undefined; + Timestamp.startTimestamp(b); + Timestamp.addUnit(b, node.unit); + if (timezone !== undefined) { + Timestamp.addTimezone(b, timezone); + } + return Timestamp.endTimestamp(b); + } + public visitInterval<T extends type.Interval>(node: T, b: Builder) { + Interval.startInterval(b); + Interval.addUnit(b, node.unit); + return Interval.endInterval(b); + } + public visitList<T extends type.List>(_node: T, b: Builder) { + List.startList(b); + return List.endList(b); + } + public visitStruct<T extends type.Struct>(_node: T, b: Builder) { + Struct.startStruct_(b); + return Struct.endStruct_(b); + } + public visitUnion<T extends type.Union>(node: T, b: Builder) { + Union.startTypeIdsVector(b, node.typeIds.length); + const typeIds = Union.createTypeIdsVector(b, node.typeIds); + Union.startUnion(b); + Union.addMode(b, node.mode); + Union.addTypeIds(b, typeIds); + return Union.endUnion(b); + } + public visitDictionary<T extends type.Dictionary>(node: T, b: Builder) { + const indexType = this.visit(node.indices, b); + DictionaryEncoding.startDictionaryEncoding(b); + DictionaryEncoding.addId(b, new Long(node.id, 0)); + DictionaryEncoding.addIsOrdered(b, node.isOrdered); + if (indexType !== undefined) { + DictionaryEncoding.addIndexType(b, indexType); + } + return DictionaryEncoding.endDictionaryEncoding(b); + } + public visitFixedSizeBinary<T extends type.FixedSizeBinary>(node: T, b: Builder) { + FixedSizeBinary.startFixedSizeBinary(b); + FixedSizeBinary.addByteWidth(b, node.byteWidth); + return FixedSizeBinary.endFixedSizeBinary(b); + } + public visitFixedSizeList<T extends type.FixedSizeList>(node: T, b: Builder) { + FixedSizeList.startFixedSizeList(b); + FixedSizeList.addListSize(b, node.listSize); + return FixedSizeList.endFixedSizeList(b); + } + public visitMap<T extends type.Map_>(node: T, b: Builder) { + Map_.startMap(b); + Map_.addKeysSorted(b, node.keysSorted); + return Map_.endMap(b); + } +} + +/** @ignore */ +export const instance = new TypeAssembler(); diff --git a/src/arrow/js/src/visitor/typecomparator.ts b/src/arrow/js/src/visitor/typecomparator.ts new file mode 100644 index 000000000..478b505f8 --- /dev/null +++ b/src/arrow/js/src/visitor/typecomparator.ts @@ -0,0 +1,280 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import { Visitor } from '../visitor'; +import { VectorType } from '../interfaces'; +import { Schema, Field } from '../schema'; +import { + DataType, Dictionary, + Bool, Null, Utf8, Binary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, + Float, Float16, Float32, Float64, + Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, + Date_, DateDay, DateMillisecond, + Interval, IntervalDayTime, IntervalYearMonth, + Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, + Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, + Union, DenseUnion, SparseUnion, +} from '../type'; + +/** @ignore */ +export interface TypeComparator extends Visitor { + visit<T extends DataType>(type: T, other?: DataType | null): other is T; + visitMany<T extends DataType>(nodes: T[], others?: DataType[] | null): boolean[]; + getVisitFn<T extends DataType>(node: VectorType<T> | Data<T> | T): (other?: DataType | null) => other is T; + visitNull <T extends Null> (type: T, other?: DataType | null): other is T; + visitBool <T extends Bool> (type: T, other?: DataType | null): other is T; + visitInt <T extends Int> (type: T, other?: DataType | null): other is T; + visitInt8 <T extends Int8> (type: T, other?: DataType | null): other is T; + visitInt16 <T extends Int16> (type: T, other?: DataType | null): other is T; + visitInt32 <T extends Int32> (type: T, other?: DataType | null): other is T; + visitInt64 <T extends Int64> (type: T, other?: DataType | null): other is T; + visitUint8 <T extends Uint8> (type: T, other?: DataType | null): other is T; + visitUint16 <T extends Uint16> (type: T, other?: DataType | null): other is T; + visitUint32 <T extends Uint32> (type: T, other?: DataType | null): other is T; + visitUint64 <T extends Uint64> (type: T, other?: DataType | null): other is T; + visitFloat <T extends Float> (type: T, other?: DataType | null): other is T; + visitFloat16 <T extends Float16> (type: T, other?: DataType | null): other is T; + visitFloat32 <T extends Float32> (type: T, other?: DataType | null): other is T; + visitFloat64 <T extends Float64> (type: T, other?: DataType | null): other is T; + visitUtf8 <T extends Utf8> (type: T, other?: DataType | null): other is T; + visitBinary <T extends Binary> (type: T, other?: DataType | null): other is T; + visitFixedSizeBinary <T extends FixedSizeBinary> (type: T, other?: DataType | null): other is T; + visitDate <T extends Date_> (type: T, other?: DataType | null): other is T; + visitDateDay <T extends DateDay> (type: T, other?: DataType | null): other is T; + visitDateMillisecond <T extends DateMillisecond> (type: T, other?: DataType | null): other is T; + visitTimestamp <T extends Timestamp> (type: T, other?: DataType | null): other is T; + visitTimestampSecond <T extends TimestampSecond> (type: T, other?: DataType | null): other is T; + visitTimestampMillisecond <T extends TimestampMillisecond> (type: T, other?: DataType | null): other is T; + visitTimestampMicrosecond <T extends TimestampMicrosecond> (type: T, other?: DataType | null): other is T; + visitTimestampNanosecond <T extends TimestampNanosecond> (type: T, other?: DataType | null): other is T; + visitTime <T extends Time> (type: T, other?: DataType | null): other is T; + visitTimeSecond <T extends TimeSecond> (type: T, other?: DataType | null): other is T; + visitTimeMillisecond <T extends TimeMillisecond> (type: T, other?: DataType | null): other is T; + visitTimeMicrosecond <T extends TimeMicrosecond> (type: T, other?: DataType | null): other is T; + visitTimeNanosecond <T extends TimeNanosecond> (type: T, other?: DataType | null): other is T; + visitDecimal <T extends Decimal> (type: T, other?: DataType | null): other is T; + visitList <T extends List> (type: T, other?: DataType | null): other is T; + visitStruct <T extends Struct> (type: T, other?: DataType | null): other is T; + visitUnion <T extends Union> (type: T, other?: DataType | null): other is T; + visitDenseUnion <T extends DenseUnion> (type: T, other?: DataType | null): other is T; + visitSparseUnion <T extends SparseUnion> (type: T, other?: DataType | null): other is T; + visitDictionary <T extends Dictionary> (type: T, other?: DataType | null): other is T; + visitInterval <T extends Interval> (type: T, other?: DataType | null): other is T; + visitIntervalDayTime <T extends IntervalDayTime> (type: T, other?: DataType | null): other is T; + visitIntervalYearMonth <T extends IntervalYearMonth> (type: T, other?: DataType | null): other is T; + visitFixedSizeList <T extends FixedSizeList> (type: T, other?: DataType | null): other is T; + visitMap <T extends Map_> (type: T, other?: DataType | null): other is T; +} + +/** @ignore */ +export class TypeComparator extends Visitor { + compareSchemas<T extends { [key: string]: DataType }>(schema: Schema<T>, other?: Schema | null): other is Schema<T> { + return (schema === other) || ( + other instanceof schema.constructor && + this.compareManyFields(schema.fields, other.fields) + ); + } + compareManyFields<T extends { [key: string]: DataType }>(fields: Field<T[keyof T]>[], others?: Field[] | null): others is Field<T[keyof T]>[] { + return (fields === others) || ( + Array.isArray(fields) && + Array.isArray(others) && + fields.length === others.length && + fields.every((f, i) => this.compareFields(f, others[i])) + ); + } + compareFields<T extends DataType = any>(field: Field<T>, other?: Field | null): other is Field<T> { + return (field === other) || ( + other instanceof field.constructor && + field.name === other.name && + field.nullable === other.nullable && + this.visit(field.type, other.type) + ); + } +} + +function compareConstructor<T extends DataType>(type: T, other?: DataType | null): other is T { + return other instanceof type.constructor; +} + +function compareAny<T extends DataType>(type: T, other?: DataType | null): other is T { + return (type === other) || compareConstructor(type, other); +} + +function compareInt<T extends Int>(type: T, other?: DataType | null): other is T { + return (type === other) || ( + compareConstructor(type, other) && + type.bitWidth === other.bitWidth && + type.isSigned === other.isSigned + ); +} + +function compareFloat<T extends Float>(type: T, other?: DataType | null): other is T { + return (type === other) || ( + compareConstructor(type, other) && + type.precision === other.precision + ); +} + +function compareFixedSizeBinary<T extends FixedSizeBinary>(type: T, other?: DataType | null): other is T { + return (type === other) || ( + compareConstructor(type, other) && + type.byteWidth === other.byteWidth + ); +} + +function compareDate<T extends Date_>(type: T, other?: DataType | null): other is T { + return (type === other) || ( + compareConstructor(type, other) && + type.unit === other.unit + ); +} + +function compareTimestamp<T extends Timestamp>(type: T, other?: DataType | null): other is T { + return (type === other) || ( + compareConstructor(type, other) && + type.unit === other.unit && + type.timezone === other.timezone + ); +} + +function compareTime<T extends Time>(type: T, other?: DataType | null): other is T { + return (type === other) || ( + compareConstructor(type, other) && + type.unit === other.unit && + type.bitWidth === other.bitWidth + ); +} + +function compareList<T extends List>(type: T, other?: DataType | null): other is T { + return (type === other) || ( + compareConstructor(type, other) && + type.children.length === other.children.length && + instance.compareManyFields(type.children, other.children) + ); +} + +function compareStruct<T extends Struct>(type: T, other?: DataType | null): other is T { + return (type === other) || ( + compareConstructor(type, other) && + type.children.length === other.children.length && + instance.compareManyFields(type.children, other.children) + ); +} + +function compareUnion<T extends Union>(type: T, other?: DataType | null): other is T { + return (type === other) || ( + compareConstructor(type, other) && + type.mode === other.mode && + type.typeIds.every((x, i) => x === other.typeIds[i]) && + instance.compareManyFields(type.children, other.children) + ); +} + +function compareDictionary<T extends Dictionary>(type: T, other?: DataType | null): other is T { + return (type === other) || ( + compareConstructor(type, other) && + type.id === other.id && + type.isOrdered === other.isOrdered && + instance.visit(<any> type.indices, other.indices) && + instance.visit(type.dictionary, other.dictionary) + ); +} + +function compareInterval<T extends Interval>(type: T, other?: DataType | null): other is T { + return (type === other) || ( + compareConstructor(type, other) && + type.unit === other.unit + ); +} + +function compareFixedSizeList<T extends FixedSizeList>(type: T, other?: DataType | null): other is T { + return (type === other) || ( + compareConstructor(type, other) && + type.listSize === other.listSize && + type.children.length === other.children.length && + instance.compareManyFields(type.children, other.children) + ); +} + +function compareMap<T extends Map_>(type: T, other?: DataType | null): other is T { + return (type === other) || ( + compareConstructor(type, other) && + type.keysSorted === other.keysSorted && + type.children.length === other.children.length && + instance.compareManyFields(type.children, other.children) + ); +} + +TypeComparator.prototype.visitNull = compareAny; +TypeComparator.prototype.visitBool = compareAny; +TypeComparator.prototype.visitInt = compareInt; +TypeComparator.prototype.visitInt8 = compareInt; +TypeComparator.prototype.visitInt16 = compareInt; +TypeComparator.prototype.visitInt32 = compareInt; +TypeComparator.prototype.visitInt64 = compareInt; +TypeComparator.prototype.visitUint8 = compareInt; +TypeComparator.prototype.visitUint16 = compareInt; +TypeComparator.prototype.visitUint32 = compareInt; +TypeComparator.prototype.visitUint64 = compareInt; +TypeComparator.prototype.visitFloat = compareFloat; +TypeComparator.prototype.visitFloat16 = compareFloat; +TypeComparator.prototype.visitFloat32 = compareFloat; +TypeComparator.prototype.visitFloat64 = compareFloat; +TypeComparator.prototype.visitUtf8 = compareAny; +TypeComparator.prototype.visitBinary = compareAny; +TypeComparator.prototype.visitFixedSizeBinary = compareFixedSizeBinary; +TypeComparator.prototype.visitDate = compareDate; +TypeComparator.prototype.visitDateDay = compareDate; +TypeComparator.prototype.visitDateMillisecond = compareDate; +TypeComparator.prototype.visitTimestamp = compareTimestamp; +TypeComparator.prototype.visitTimestampSecond = compareTimestamp; +TypeComparator.prototype.visitTimestampMillisecond = compareTimestamp; +TypeComparator.prototype.visitTimestampMicrosecond = compareTimestamp; +TypeComparator.prototype.visitTimestampNanosecond = compareTimestamp; +TypeComparator.prototype.visitTime = compareTime; +TypeComparator.prototype.visitTimeSecond = compareTime; +TypeComparator.prototype.visitTimeMillisecond = compareTime; +TypeComparator.prototype.visitTimeMicrosecond = compareTime; +TypeComparator.prototype.visitTimeNanosecond = compareTime; +TypeComparator.prototype.visitDecimal = compareAny; +TypeComparator.prototype.visitList = compareList; +TypeComparator.prototype.visitStruct = compareStruct; +TypeComparator.prototype.visitUnion = compareUnion; +TypeComparator.prototype.visitDenseUnion = compareUnion; +TypeComparator.prototype.visitSparseUnion = compareUnion; +TypeComparator.prototype.visitDictionary = compareDictionary; +TypeComparator.prototype.visitInterval = compareInterval; +TypeComparator.prototype.visitIntervalDayTime = compareInterval; +TypeComparator.prototype.visitIntervalYearMonth = compareInterval; +TypeComparator.prototype.visitFixedSizeList = compareFixedSizeList; +TypeComparator.prototype.visitMap = compareMap; + +/** @ignore */ +export const instance = new TypeComparator(); + +export function compareSchemas<T extends { [key: string]: DataType }>(schema: Schema<T>, other?: Schema | null): other is Schema<T> { + return instance.compareSchemas(schema, other); +} + +export function compareFields<T extends DataType = any>(field: Field<T>, other?: Field | null): other is Field<T> { + return instance.compareFields(field, other); +} + +export function compareTypes<A extends DataType = any>(type: A, other?: DataType): other is A { + return instance.visit(type, other); +} diff --git a/src/arrow/js/src/visitor/typector.ts b/src/arrow/js/src/visitor/typector.ts new file mode 100644 index 000000000..9d0a9f17d --- /dev/null +++ b/src/arrow/js/src/visitor/typector.ts @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import { Type } from '../enum'; +import * as type from '../type'; +import { DataType } from '../type'; +import { Visitor } from '../visitor'; +import { VectorType } from '../interfaces'; +import { DataTypeCtor } from '../interfaces'; + +/** @ignore */ +export interface GetDataTypeConstructor extends Visitor { + visit<T extends Type>(node: T): DataTypeCtor<T>; + visitMany<T extends Type>(nodes: T[]): DataTypeCtor<T>[]; + getVisitFn<T extends Type>(node: T): () => DataTypeCtor<T>; + getVisitFn<T extends DataType>(node: VectorType<T> | Data<T> | T): () => DataTypeCtor<T>; +} + +/** @ignore */ +export class GetDataTypeConstructor extends Visitor { + public visitNull () { return type.Null; } + public visitBool () { return type.Bool; } + public visitInt () { return type.Int; } + public visitInt8 () { return type.Int8; } + public visitInt16 () { return type.Int16; } + public visitInt32 () { return type.Int32; } + public visitInt64 () { return type.Int64; } + public visitUint8 () { return type.Uint8; } + public visitUint16 () { return type.Uint16; } + public visitUint32 () { return type.Uint32; } + public visitUint64 () { return type.Uint64; } + public visitFloat () { return type.Float; } + public visitFloat16 () { return type.Float16; } + public visitFloat32 () { return type.Float32; } + public visitFloat64 () { return type.Float64; } + public visitUtf8 () { return type.Utf8; } + public visitBinary () { return type.Binary; } + public visitFixedSizeBinary () { return type.FixedSizeBinary; } + public visitDate () { return type.Date_; } + public visitDateDay () { return type.DateDay; } + public visitDateMillisecond () { return type.DateMillisecond; } + public visitTimestamp () { return type.Timestamp; } + public visitTimestampSecond () { return type.TimestampSecond; } + public visitTimestampMillisecond () { return type.TimestampMillisecond; } + public visitTimestampMicrosecond () { return type.TimestampMicrosecond; } + public visitTimestampNanosecond () { return type.TimestampNanosecond; } + public visitTime () { return type.Time; } + public visitTimeSecond () { return type.TimeSecond; } + public visitTimeMillisecond () { return type.TimeMillisecond; } + public visitTimeMicrosecond () { return type.TimeMicrosecond; } + public visitTimeNanosecond () { return type.TimeNanosecond; } + public visitDecimal () { return type.Decimal; } + public visitList () { return type.List; } + public visitStruct () { return type.Struct; } + public visitUnion () { return type.Union; } + public visitDenseUnion () { return type.DenseUnion; } + public visitSparseUnion () { return type.SparseUnion; } + public visitDictionary () { return type.Dictionary; } + public visitInterval () { return type.Interval; } + public visitIntervalDayTime () { return type.IntervalDayTime; } + public visitIntervalYearMonth () { return type.IntervalYearMonth; } + public visitFixedSizeList () { return type.FixedSizeList; } + public visitMap () { return type.Map_; } +} + +/** @ignore */ +export const instance = new GetDataTypeConstructor(); diff --git a/src/arrow/js/src/visitor/vectorassembler.ts b/src/arrow/js/src/visitor/vectorassembler.ts new file mode 100644 index 000000000..e324bc02e --- /dev/null +++ b/src/arrow/js/src/visitor/vectorassembler.ts @@ -0,0 +1,234 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import { Vector } from '../vector'; +import { Visitor } from '../visitor'; +import { Type, UnionMode } from '../enum'; +import { RecordBatch } from '../recordbatch'; +import { VectorType as V } from '../interfaces'; +import { rebaseValueOffsets } from '../util/buffer'; +import { packBools, truncateBitmap } from '../util/bit'; +import { selectVectorChildrenArgs } from '../util/args'; +import { BufferRegion, FieldNode } from '../ipc/metadata/message'; +import { + DataType, Dictionary, + Float, Int, Date_, Interval, Time, Timestamp, Union, + Bool, Null, Utf8, Binary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, +} from '../type'; + +/** @ignore */ +export interface VectorAssembler extends Visitor { + visit<T extends Vector>(node: T): this; + visitMany<T extends Vector>(nodes: T[]): this[]; + getVisitFn<T extends Type>(node: T): (vector: V<T>) => this; + getVisitFn<T extends DataType>(node: V<T> | Data<T> | T): (vector: V<T>) => this; + + visitBool <T extends Bool> (vector: V<T>): this; + visitInt <T extends Int> (vector: V<T>): this; + visitFloat <T extends Float> (vector: V<T>): this; + visitUtf8 <T extends Utf8> (vector: V<T>): this; + visitBinary <T extends Binary> (vector: V<T>): this; + visitFixedSizeBinary <T extends FixedSizeBinary> (vector: V<T>): this; + visitDate <T extends Date_> (vector: V<T>): this; + visitTimestamp <T extends Timestamp> (vector: V<T>): this; + visitTime <T extends Time> (vector: V<T>): this; + visitDecimal <T extends Decimal> (vector: V<T>): this; + visitList <T extends List> (vector: V<T>): this; + visitStruct <T extends Struct> (vector: V<T>): this; + visitUnion <T extends Union> (vector: V<T>): this; + visitInterval <T extends Interval> (vector: V<T>): this; + visitFixedSizeList <T extends FixedSizeList> (vector: V<T>): this; + visitMap <T extends Map_> (vector: V<T>): this; +} + +/** @ignore */ +export class VectorAssembler extends Visitor { + + /** @nocollapse */ + public static assemble<T extends Vector | RecordBatch>(...args: (T | T[])[]) { + const assembler = new VectorAssembler(); + const vectorChildren = selectVectorChildrenArgs(RecordBatch, args); + const [assembleResult = assembler] = assembler.visitMany(vectorChildren); + return assembleResult; + } + + private constructor() { super(); } + + public visit<T extends Vector>(vector: T): this { + if (!DataType.isDictionary(vector.type)) { + const { data, length, nullCount } = vector; + if (length > 2147483647) { + /* istanbul ignore next */ + throw new RangeError('Cannot write arrays larger than 2^31 - 1 in length'); + } + if (!DataType.isNull(vector.type)) { + addBuffer.call(this, nullCount <= 0 + ? new Uint8Array(0) // placeholder validity buffer + : truncateBitmap(data.offset, length, data.nullBitmap) + ); + } + this.nodes.push(new FieldNode(length, nullCount)); + } + return super.visit(vector); + } + + public visitNull<T extends Null>(_nullV: V<T>) { + return this; + } + public visitDictionary<T extends Dictionary>(vector: V<T>) { + // Assemble the indices here, Dictionary assembled separately. + return this.visit(vector.indices); + } + + public get nodes() { return this._nodes; } + public get buffers() { return this._buffers; } + public get byteLength() { return this._byteLength; } + public get bufferRegions() { return this._bufferRegions; } + + protected _byteLength = 0; + protected _nodes: FieldNode[] = []; + protected _buffers: ArrayBufferView[] = []; + protected _bufferRegions: BufferRegion[] = []; +} + +/** @ignore */ +function addBuffer(this: VectorAssembler, values: ArrayBufferView) { + const byteLength = (values.byteLength + 7) & ~7; // Round up to a multiple of 8 + this.buffers.push(values); + this.bufferRegions.push(new BufferRegion(this._byteLength, byteLength)); + this._byteLength += byteLength; + return this; +} + +/** @ignore */ +function assembleUnion<T extends Union>(this: VectorAssembler, vector: V<T>) { + const { type, length, typeIds, valueOffsets } = vector; + // All Union Vectors have a typeIds buffer + addBuffer.call(this, typeIds); + // If this is a Sparse Union, treat it like all other Nested types + if (type.mode === UnionMode.Sparse) { + return assembleNestedVector.call(this, vector); + } else if (type.mode === UnionMode.Dense) { + // If this is a Dense Union, add the valueOffsets buffer and potentially slice the children + if (vector.offset <= 0) { + // If the Vector hasn't been sliced, write the existing valueOffsets + addBuffer.call(this, valueOffsets); + // We can treat this like all other Nested types + return assembleNestedVector.call(this, vector); + } else { + // A sliced Dense Union is an unpleasant case. Because the offsets are different for + // each child vector, we need to "rebase" the valueOffsets for each child + // Union typeIds are not necessary 0-indexed + const maxChildTypeId = typeIds.reduce((x, y) => Math.max(x, y), typeIds[0]); + const childLengths = new Int32Array(maxChildTypeId + 1); + // Set all to -1 to indicate that we haven't observed a first occurrence of a particular child yet + const childOffsets = new Int32Array(maxChildTypeId + 1).fill(-1); + const shiftedOffsets = new Int32Array(length); + // If we have a non-zero offset, then the value offsets do not start at + // zero. We must a) create a new offsets array with shifted offsets and + // b) slice the values array accordingly + const unshiftedOffsets = rebaseValueOffsets(-valueOffsets[0], length, valueOffsets); + for (let typeId, shift, index = -1; ++index < length;) { + if ((shift = childOffsets[typeId = typeIds[index]]) === -1) { + shift = childOffsets[typeId] = unshiftedOffsets[typeId]; + } + shiftedOffsets[index] = unshiftedOffsets[index] - shift; + ++childLengths[typeId]; + } + addBuffer.call(this, shiftedOffsets); + // Slice and visit children accordingly + for (let child: Vector | null, childIndex = -1, numChildren = type.children.length; ++childIndex < numChildren;) { + if (child = vector.getChildAt(childIndex)) { + const typeId = type.typeIds[childIndex]; + const childLength = Math.min(length, childLengths[typeId]); + this.visit(child.slice(childOffsets[typeId], childLength)); + } + } + } + } + return this; +} + +/** @ignore */ +function assembleBoolVector<T extends Bool>(this: VectorAssembler, vector: V<T>) { + // Bool vector is a special case of FlatVector, as its data buffer needs to stay packed + let values: Uint8Array; + if (vector.nullCount >= vector.length) { + // If all values are null, just insert a placeholder empty data buffer (fastest path) + return addBuffer.call(this, new Uint8Array(0)); + } else if ((values = vector.values) instanceof Uint8Array) { + // If values is already a Uint8Array, slice the bitmap (fast path) + return addBuffer.call(this, truncateBitmap(vector.offset, vector.length, values)); + } + // Otherwise if the underlying data *isn't* a Uint8Array, enumerate the + // values as bools and re-pack them into a Uint8Array. This code isn't + // reachable unless you're trying to manipulate the Data internals, + // we we're only doing this for safety. + /* istanbul ignore next */ + return addBuffer.call(this, packBools(vector)); +} + +/** @ignore */ +function assembleFlatVector<T extends Int | Float | FixedSizeBinary | Date_ | Timestamp | Time | Decimal | Interval>(this: VectorAssembler, vector: V<T>) { + return addBuffer.call(this, vector.values.subarray(0, vector.length * vector.stride)); +} + +/** @ignore */ +function assembleFlatListVector<T extends Utf8 | Binary>(this: VectorAssembler, vector: V<T>) { + const { length, values, valueOffsets } = vector; + const firstOffset = valueOffsets[0]; + const lastOffset = valueOffsets[length]; + const byteLength = Math.min(lastOffset - firstOffset, values.byteLength - firstOffset); + // Push in the order FlatList types read their buffers + addBuffer.call(this, rebaseValueOffsets(-valueOffsets[0], length, valueOffsets)); // valueOffsets buffer first + addBuffer.call(this, values.subarray(firstOffset, firstOffset + byteLength)); // sliced values buffer second + return this; +} + +/** @ignore */ +function assembleListVector<T extends Map_ | List | FixedSizeList>(this: VectorAssembler, vector: V<T>) { + const { length, valueOffsets } = vector; + // If we have valueOffsets (MapVector, ListVector), push that buffer first + if (valueOffsets) { + addBuffer.call(this, rebaseValueOffsets(valueOffsets[0], length, valueOffsets)); + } + // Then insert the List's values child + return this.visit(vector.getChildAt(0)!); +} + +/** @ignore */ +function assembleNestedVector<T extends Struct | Union>(this: VectorAssembler, vector: V<T>) { + return this.visitMany(vector.type.children.map((_, i) => vector.getChildAt(i)!).filter(Boolean))[0]; +} + +VectorAssembler.prototype.visitBool = assembleBoolVector; +VectorAssembler.prototype.visitInt = assembleFlatVector; +VectorAssembler.prototype.visitFloat = assembleFlatVector; +VectorAssembler.prototype.visitUtf8 = assembleFlatListVector; +VectorAssembler.prototype.visitBinary = assembleFlatListVector; +VectorAssembler.prototype.visitFixedSizeBinary = assembleFlatVector; +VectorAssembler.prototype.visitDate = assembleFlatVector; +VectorAssembler.prototype.visitTimestamp = assembleFlatVector; +VectorAssembler.prototype.visitTime = assembleFlatVector; +VectorAssembler.prototype.visitDecimal = assembleFlatVector; +VectorAssembler.prototype.visitList = assembleListVector; +VectorAssembler.prototype.visitStruct = assembleNestedVector; +VectorAssembler.prototype.visitUnion = assembleUnion; +VectorAssembler.prototype.visitInterval = assembleFlatVector; +VectorAssembler.prototype.visitFixedSizeList = assembleListVector; +VectorAssembler.prototype.visitMap = assembleListVector; diff --git a/src/arrow/js/src/visitor/vectorctor.ts b/src/arrow/js/src/visitor/vectorctor.ts new file mode 100644 index 000000000..5db268c00 --- /dev/null +++ b/src/arrow/js/src/visitor/vectorctor.ts @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import { Type } from '../enum'; +import { DataType } from '../type'; +import { Visitor } from '../visitor'; +import { VectorType, VectorCtor } from '../interfaces'; + +import { BinaryVector } from '../vector/binary'; +import { BoolVector } from '../vector/bool'; +import { DateVector, DateDayVector, DateMillisecondVector } from '../vector/date'; +import { DecimalVector } from '../vector/decimal'; +import { DictionaryVector } from '../vector/dictionary'; +import { FixedSizeBinaryVector } from '../vector/fixedsizebinary'; +import { FixedSizeListVector } from '../vector/fixedsizelist'; +import { FloatVector, Float16Vector, Float32Vector, Float64Vector } from '../vector/float'; +import { IntervalVector, IntervalDayTimeVector, IntervalYearMonthVector } from '../vector/interval'; +import { IntVector, Int8Vector, Int16Vector, Int32Vector, Int64Vector, Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector } from '../vector/int'; +import { ListVector } from '../vector/list'; +import { MapVector } from '../vector/map'; +import { NullVector } from '../vector/null'; +import { StructVector } from '../vector/struct'; +import { TimestampVector, TimestampSecondVector, TimestampMillisecondVector, TimestampMicrosecondVector, TimestampNanosecondVector } from '../vector/timestamp'; +import { TimeVector, TimeSecondVector, TimeMillisecondVector, TimeMicrosecondVector, TimeNanosecondVector } from '../vector/time'; +import { UnionVector, DenseUnionVector, SparseUnionVector } from '../vector/union'; +import { Utf8Vector } from '../vector/utf8'; + +/** @ignore */ +export interface GetVectorConstructor extends Visitor { + visit<T extends Type>(node: T): VectorCtor<T>; + visitMany <T extends Type>(nodes: T[]): VectorCtor<T>[]; + getVisitFn<T extends Type>(node: T): () => VectorCtor<T>; + getVisitFn<T extends DataType>(node: VectorType<T> | Data<T> | T): () => VectorCtor<T>; +} + +/** @ignore */ +export class GetVectorConstructor extends Visitor { + public visitNull () { return NullVector; } + public visitBool () { return BoolVector; } + public visitInt () { return IntVector; } + public visitInt8 () { return Int8Vector; } + public visitInt16 () { return Int16Vector; } + public visitInt32 () { return Int32Vector; } + public visitInt64 () { return Int64Vector; } + public visitUint8 () { return Uint8Vector; } + public visitUint16 () { return Uint16Vector; } + public visitUint32 () { return Uint32Vector; } + public visitUint64 () { return Uint64Vector; } + public visitFloat () { return FloatVector; } + public visitFloat16 () { return Float16Vector; } + public visitFloat32 () { return Float32Vector; } + public visitFloat64 () { return Float64Vector; } + public visitUtf8 () { return Utf8Vector; } + public visitBinary () { return BinaryVector; } + public visitFixedSizeBinary () { return FixedSizeBinaryVector; } + public visitDate () { return DateVector; } + public visitDateDay () { return DateDayVector; } + public visitDateMillisecond () { return DateMillisecondVector; } + public visitTimestamp () { return TimestampVector; } + public visitTimestampSecond () { return TimestampSecondVector; } + public visitTimestampMillisecond () { return TimestampMillisecondVector; } + public visitTimestampMicrosecond () { return TimestampMicrosecondVector; } + public visitTimestampNanosecond () { return TimestampNanosecondVector; } + public visitTime () { return TimeVector; } + public visitTimeSecond () { return TimeSecondVector; } + public visitTimeMillisecond () { return TimeMillisecondVector; } + public visitTimeMicrosecond () { return TimeMicrosecondVector; } + public visitTimeNanosecond () { return TimeNanosecondVector; } + public visitDecimal () { return DecimalVector; } + public visitList () { return ListVector; } + public visitStruct () { return StructVector; } + public visitUnion () { return UnionVector; } + public visitDenseUnion () { return DenseUnionVector; } + public visitSparseUnion () { return SparseUnionVector; } + public visitDictionary () { return DictionaryVector; } + public visitInterval () { return IntervalVector; } + public visitIntervalDayTime () { return IntervalDayTimeVector; } + public visitIntervalYearMonth () { return IntervalYearMonthVector; } + public visitFixedSizeList () { return FixedSizeListVector; } + public visitMap () { return MapVector; } +} + +/** @ignore */ +export const instance = new GetVectorConstructor(); diff --git a/src/arrow/js/src/visitor/vectorloader.ts b/src/arrow/js/src/visitor/vectorloader.ts new file mode 100644 index 000000000..0a7bb41d8 --- /dev/null +++ b/src/arrow/js/src/visitor/vectorloader.ts @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from '../data'; +import * as type from '../type'; +import { Field } from '../schema'; +import { Vector } from '../vector'; +import { DataType } from '../type'; +import { Visitor } from '../visitor'; +import { packBools } from '../util/bit'; +import { encodeUtf8 } from '../util/utf8'; +import { Int64, Int128 } from '../util/int'; +import { UnionMode, DateUnit } from '../enum'; +import { toArrayBufferView } from '../util/buffer'; +import { BufferRegion, FieldNode } from '../ipc/metadata/message'; + +/** @ignore */ +export interface VectorLoader extends Visitor { + visit<T extends DataType>(node: Field<T> | T): Data<T>; + visitMany<T extends DataType>(nodes: (Field<T> | T)[]): Data<T>[]; +} + +/** @ignore */ +export class VectorLoader extends Visitor { + private bytes: Uint8Array; + private nodes: FieldNode[]; + private nodesIndex = -1; + private buffers: BufferRegion[]; + private buffersIndex = -1; + private dictionaries: Map<number, Vector<any>>; + constructor(bytes: Uint8Array, nodes: FieldNode[], buffers: BufferRegion[], dictionaries: Map<number, Vector<any>>) { + super(); + this.bytes = bytes; + this.nodes = nodes; + this.buffers = buffers; + this.dictionaries = dictionaries; + } + + public visit<T extends DataType>(node: Field<T> | T): Data<T> { + return super.visit(node instanceof Field ? node.type : node); + } + + public visitNull <T extends type.Null> (type: T, { length, } = this.nextFieldNode()) { return Data.Null(type, 0, length); } + public visitBool <T extends type.Bool> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Bool(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type)); } + public visitInt <T extends type.Int> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Int(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type)); } + public visitFloat <T extends type.Float> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Float(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type)); } + public visitUtf8 <T extends type.Utf8> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Utf8(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readOffsets(type), this.readData(type)); } + public visitBinary <T extends type.Binary> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Binary(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readOffsets(type), this.readData(type)); } + public visitFixedSizeBinary <T extends type.FixedSizeBinary> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.FixedSizeBinary(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type)); } + public visitDate <T extends type.Date_> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Date(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type)); } + public visitTimestamp <T extends type.Timestamp> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Timestamp(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type)); } + public visitTime <T extends type.Time> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Time(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type)); } + public visitDecimal <T extends type.Decimal> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Decimal(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type)); } + public visitList <T extends type.List> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.List(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readOffsets(type), this.visit(type.children[0])); } + public visitStruct <T extends type.Struct> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Struct(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.visitMany(type.children)); } + public visitUnion <T extends type.Union> (type: T ) { return type.mode === UnionMode.Sparse ? this.visitSparseUnion(type as type.SparseUnion) : this.visitDenseUnion(type as type.DenseUnion); } + public visitDenseUnion <T extends type.DenseUnion> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Union(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readTypeIds(type), this.readOffsets(type), this.visitMany(type.children)); } + public visitSparseUnion <T extends type.SparseUnion> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Union(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readTypeIds(type), this.visitMany(type.children)); } + public visitDictionary <T extends type.Dictionary> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Dictionary(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type.indices), this.readDictionary(type)); } + public visitInterval <T extends type.Interval> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Interval(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readData(type)); } + public visitFixedSizeList <T extends type.FixedSizeList> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.FixedSizeList(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.visit(type.children[0])); } + public visitMap <T extends type.Map_> (type: T, { length, nullCount } = this.nextFieldNode()) { return Data.Map(type, 0, length, nullCount, this.readNullBitmap(type, nullCount), this.readOffsets(type), this.visit(type.children[0])); } + + protected nextFieldNode() { return this.nodes[++this.nodesIndex]; } + protected nextBufferRange() { return this.buffers[++this.buffersIndex]; } + protected readNullBitmap<T extends DataType>(type: T, nullCount: number, buffer = this.nextBufferRange()) { + return nullCount > 0 && this.readData(type, buffer) || new Uint8Array(0); + } + protected readOffsets<T extends DataType>(type: T, buffer?: BufferRegion) { return this.readData(type, buffer); } + protected readTypeIds<T extends DataType>(type: T, buffer?: BufferRegion) { return this.readData(type, buffer); } + protected readData<T extends DataType>(_type: T, { length, offset } = this.nextBufferRange()) { + return this.bytes.subarray(offset, offset + length); + } + protected readDictionary<T extends type.Dictionary>(type: T): Vector<T['dictionary']> { + return this.dictionaries.get(type.id)!; + } +} + +/** @ignore */ +export class JSONVectorLoader extends VectorLoader { + private sources: any[][]; + constructor(sources: any[][], nodes: FieldNode[], buffers: BufferRegion[], dictionaries: Map<number, Vector<any>>) { + super(new Uint8Array(0), nodes, buffers, dictionaries); + this.sources = sources; + } + protected readNullBitmap<T extends DataType>(_type: T, nullCount: number, { offset } = this.nextBufferRange()) { + return nullCount <= 0 ? new Uint8Array(0) : packBools(this.sources[offset]); + } + protected readOffsets<T extends DataType>(_type: T, { offset } = this.nextBufferRange()) { + return toArrayBufferView(Uint8Array, toArrayBufferView(Int32Array, this.sources[offset])); + } + protected readTypeIds<T extends DataType>(type: T, { offset } = this.nextBufferRange()) { + return toArrayBufferView(Uint8Array, toArrayBufferView(type.ArrayType, this.sources[offset])); + } + protected readData<T extends DataType>(type: T, { offset } = this.nextBufferRange()) { + const { sources } = this; + if (DataType.isTimestamp(type)) { + return toArrayBufferView(Uint8Array, Int64.convertArray(sources[offset] as string[])); + } else if ((DataType.isInt(type) || DataType.isTime(type)) && type.bitWidth === 64) { + return toArrayBufferView(Uint8Array, Int64.convertArray(sources[offset] as string[])); + } else if (DataType.isDate(type) && type.unit === DateUnit.MILLISECOND) { + return toArrayBufferView(Uint8Array, Int64.convertArray(sources[offset] as string[])); + } else if (DataType.isDecimal(type)) { + return toArrayBufferView(Uint8Array, Int128.convertArray(sources[offset] as string[])); + } else if (DataType.isBinary(type) || DataType.isFixedSizeBinary(type)) { + return binaryDataFromJSON(sources[offset] as string[]); + } else if (DataType.isBool(type)) { + return packBools(sources[offset] as number[]); + } else if (DataType.isUtf8(type)) { + return encodeUtf8((sources[offset] as string[]).join('')); + } + return toArrayBufferView(Uint8Array, toArrayBufferView(type.ArrayType, sources[offset].map((x) => +x))); + } +} + +/** @ignore */ +function binaryDataFromJSON(values: string[]) { + // "DATA": ["49BC7D5B6C47D2","3F5FB6D9322026"] + // There are definitely more efficient ways to do this... but it gets the + // job done. + const joined = values.join(''); + const data = new Uint8Array(joined.length / 2); + for (let i = 0; i < joined.length; i += 2) { + data[i >> 1] = parseInt(joined.substr(i, 2), 16); + } + return data; +} diff --git a/src/arrow/js/test/.eslintrc.js b/src/arrow/js/test/.eslintrc.js new file mode 100644 index 000000000..311a356e2 --- /dev/null +++ b/src/arrow/js/test/.eslintrc.js @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + rules: { + "@typescript-eslint/no-require-imports": "off", + "@typescript-eslint/no-inferrable-types": "off", + "@typescript-eslint/naming-convention": "off", + "prefer-const": "off", + "max-len": "off", + + "jest/no-export": "off", + "jest/valid-title": "off", + "jest/expect-expect": "off", + "jest/no-conditional-expect": "off", + }, +}; diff --git a/src/arrow/js/test/Arrow.ts b/src/arrow/js/test/Arrow.ts new file mode 100644 index 000000000..de2bc58c7 --- /dev/null +++ b/src/arrow/js/test/Arrow.ts @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import 'web-streams-polyfill'; + +export * from 'apache-arrow'; diff --git a/src/arrow/js/test/data/tables.ts b/src/arrow/js/test/data/tables.ts new file mode 100644 index 000000000..6ce2c861d --- /dev/null +++ b/src/arrow/js/test/data/tables.ts @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { vecs } from '../generate-test-data'; +import * as generate from '../generate-test-data'; +import { Schema, Field, Dictionary } from '../Arrow'; + +const listVectorGeneratorNames = ['list', 'fixedSizeList']; +const nestedVectorGeneratorNames = [ 'struct', 'denseUnion', 'sparseUnion', 'map' ]; +const dictionaryKeyGeneratorNames = ['int8' ,'int16' ,'int32' ,'uint8' ,'uint16' ,'uint32']; +const valueVectorGeneratorNames = [ + 'null_', 'bool', 'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', + 'float16', 'float32', 'float64', 'utf8', 'binary', 'fixedSizeBinary', 'dateDay', 'dateMillisecond', + 'timestampSecond', 'timestampMillisecond', 'timestampMicrosecond', 'timestampNanosecond', + 'timeSecond', 'timeMillisecond', 'timeMicrosecond', 'timeNanosecond', 'decimal', + 'dictionary', 'intervalDayTime', 'intervalYearMonth' +]; + +const vectorGeneratorNames = [...valueVectorGeneratorNames, ...listVectorGeneratorNames, ...nestedVectorGeneratorNames]; + +export function* generateRandomTables(batchLengths = [1000, 2000, 3000], minCols = 1, maxCols = 5) { + + let numCols = 0; + let allNames = shuffle(vectorGeneratorNames); + + do { + numCols = Math.max(Math.min( + Math.random() * maxCols | 0, allNames.length), minCols); + + let names = allNames.slice(0, numCols); + let types = names.map((fn) => vecs[fn](0).vector.type); + let schema = new Schema(names.map((name, i) => new Field(name, types[i]))); + + yield generate.table(batchLengths, schema).table; + + } while ((allNames = allNames.slice(numCols)).length > 0); +} + +/** + * Yields a series of tables containing a single Dictionary-encoded column. + * Each yielded table will be a unique combination of dictionary and indexType, + * such that consuming all tables ensures all Arrow types dictionary-encode. + * + * @param batchLengths number[] Number and length of recordbatches to generate + */ +export function* generateDictionaryTables(batchLengths = [100, 200, 300]) { + for (const dictName of valueVectorGeneratorNames) { + if (dictName === 'dictionary') { continue; } + const dictionary = vecs[dictName](100).vector; + for (const keys of dictionaryKeyGeneratorNames) { + const valsType = dictionary.type; + const keysType = vecs[keys](0).vector.type; + const dictType = new Dictionary(valsType, keysType); + const schema = new Schema([new Field(`dict[${keys}]`, dictType, true)]); + yield generate.table(batchLengths, schema).table; + } + } +} + +function shuffle(input: any[]) { + const result = input.slice(); + let j, tmp, i = result.length; + while (--i > 0) { + j = (Math.random() * (i + 1)) | 0; + tmp = result[i]; + result[i] = result[j]; + result[j] = tmp; + } + return result; +} diff --git a/src/arrow/js/test/generate-test-data.ts b/src/arrow/js/test/generate-test-data.ts new file mode 100644 index 000000000..030176e62 --- /dev/null +++ b/src/arrow/js/test/generate-test-data.ts @@ -0,0 +1,720 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import randomatic from 'randomatic'; +import { VectorType as V } from 'apache-arrow/interfaces'; + +import { + Data, Vector, Visitor, DataType, + Table, Schema, Field, RecordBatch, + Null, + Bool, + Int, Int8, Int16, Int32, Int64, Uint8, Uint16, Uint32, Uint64, + Float, Float16, Float32, Float64, + Utf8, + Binary, + FixedSizeBinary, + Date_, DateDay, DateMillisecond, + Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, + Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, + Decimal, + List, + Struct, + Union, DenseUnion, SparseUnion, + Dictionary, + Interval, IntervalDayTime, IntervalYearMonth, + FixedSizeList, + Map_, + DateUnit, TimeUnit, UnionMode, + util +} from './Arrow'; + +type TKeys = Int8 | Int16 | Int32 | Uint8 | Uint16 | Uint32; + +interface TestDataVectorGenerator extends Visitor { + + visit<T extends Null> (type: T, length?: number): GeneratedVector<V<T>>; + visit<T extends Bool> (type: T, length?: number, nullCount?: number): GeneratedVector<V<T>>; + visit<T extends Int> (type: T, length?: number, nullCount?: number): GeneratedVector<V<T>>; + visit<T extends Float> (type: T, length?: number, nullCount?: number): GeneratedVector<V<T>>; + visit<T extends Utf8> (type: T, length?: number, nullCount?: number): GeneratedVector<V<T>>; + visit<T extends Binary> (type: T, length?: number, nullCount?: number): GeneratedVector<V<T>>; + visit<T extends FixedSizeBinary> (type: T, length?: number, nullCount?: number): GeneratedVector<V<T>>; + visit<T extends Date_> (type: T, length?: number, nullCount?: number): GeneratedVector<V<T>>; + visit<T extends Timestamp> (type: T, length?: number, nullCount?: number): GeneratedVector<V<T>>; + visit<T extends Time> (type: T, length?: number, nullCount?: number): GeneratedVector<V<T>>; + visit<T extends Decimal> (type: T, length?: number, nullCount?: number): GeneratedVector<V<T>>; + visit<T extends Interval> (type: T, length?: number, nullCount?: number): GeneratedVector<V<T>>; + visit<T extends List> (type: T, length?: number, nullCount?: number, child?: Vector): GeneratedVector<V<T>>; + visit<T extends FixedSizeList> (type: T, length?: number, nullCount?: number, child?: Vector): GeneratedVector<V<T>>; + visit<T extends Dictionary> (type: T, length?: number, nullCount?: number, dictionary?: Vector): GeneratedVector<V<T>>; + visit<T extends Union> (type: T, length?: number, nullCount?: number, children?: Vector[]): GeneratedVector<V<T>>; + visit<T extends Struct> (type: T, length?: number, nullCount?: number, children?: Vector[]): GeneratedVector<V<T>>; + visit<T extends Map_> (type: T, length?: number, nullCount?: number, child?: Vector): GeneratedVector<V<T>>; + visit<T extends DataType> (type: T, length?: number, ...args: any[]): GeneratedVector<V<T>>; + + visitNull: typeof generateNull; + visitBool: typeof generateBool; + visitInt: typeof generateInt; + visitFloat: typeof generateFloat; + visitUtf8: typeof generateUtf8; + visitBinary: typeof generateBinary; + visitFixedSizeBinary: typeof generateFixedSizeBinary; + visitDate: typeof generateDate; + visitTimestamp: typeof generateTimestamp; + visitTime: typeof generateTime; + visitDecimal: typeof generateDecimal; + visitList: typeof generateList; + visitStruct: typeof generateStruct; + visitUnion: typeof generateUnion; + visitDictionary: typeof generateDictionary; + visitInterval: typeof generateInterval; + visitFixedSizeList: typeof generateFixedSizeList; + visitMap: typeof generateMap; +} + +class TestDataVectorGenerator extends Visitor {} + +TestDataVectorGenerator.prototype.visitNull = generateNull; +TestDataVectorGenerator.prototype.visitBool = generateBool; +TestDataVectorGenerator.prototype.visitInt = generateInt; +TestDataVectorGenerator.prototype.visitFloat = generateFloat; +TestDataVectorGenerator.prototype.visitUtf8 = generateUtf8; +TestDataVectorGenerator.prototype.visitBinary = generateBinary; +TestDataVectorGenerator.prototype.visitFixedSizeBinary = generateFixedSizeBinary; +TestDataVectorGenerator.prototype.visitDate = generateDate; +TestDataVectorGenerator.prototype.visitTimestamp = generateTimestamp; +TestDataVectorGenerator.prototype.visitTime = generateTime; +TestDataVectorGenerator.prototype.visitDecimal = generateDecimal; +TestDataVectorGenerator.prototype.visitList = generateList; +TestDataVectorGenerator.prototype.visitStruct = generateStruct; +TestDataVectorGenerator.prototype.visitUnion = generateUnion; +TestDataVectorGenerator.prototype.visitDictionary = generateDictionary; +TestDataVectorGenerator.prototype.visitInterval = generateInterval; +TestDataVectorGenerator.prototype.visitFixedSizeList = generateFixedSizeList; +TestDataVectorGenerator.prototype.visitMap = generateMap; + +const vectorGenerator = new TestDataVectorGenerator(); + +const defaultListChild = new Field('list[Int32]', new Int32()); + +const defaultRecordBatchChildren = () => [ + new Field('i32', new Int32()), + new Field('f32', new Float32()), + new Field('dict', new Dictionary(new Utf8(), new Int32())) +]; + +const defaultStructChildren = () => [ + new Field('struct[0]', new Int32()), + new Field('struct[1]', new Utf8()), + new Field('struct[2]', new List(new Field('list[DateDay]', new DateDay()))) +]; + +const defaultMapChild = () => [ + new Field('', new Struct<{ key: Utf8; value: Float32 }>([ + new Field('key', new Utf8()), + new Field('value', new Float32()) + ])) +][0]; + +const defaultUnionChildren = () => [ + new Field('union[0]', new Float64()), + new Field('union[1]', new Dictionary(new Uint32(), new Int32())), + new Field('union[2]', new Map_(defaultMapChild())) +]; + +export interface GeneratedTable { + table: Table; + rows: () => any[][]; + cols: () => any[][]; + keys: () => number[][]; + rowBatches: (() => any[][])[]; + colBatches: (() => any[][])[]; + keyBatches: (() => number[][])[]; +} + +export interface GeneratedRecordBatch { + recordBatch: RecordBatch; + rows: () => any[][]; + cols: () => any[][]; + keys: () => number[][]; +} + +export type GeneratedVector<TVec extends Vector = Vector> = { + vector: TVec; + keys?: number[]; + values: () => (TVec['TValue'] | null)[]; +}; + +export const table = (lengths = [100], schema: Schema = new Schema(defaultRecordBatchChildren(), new Map([['foo', 'bar']]))): GeneratedTable => { + const generated = lengths.map((length) => recordBatch(length, schema)); + const rowBatches = generated.map(({ rows }) => rows); + const colBatches = generated.map(({ cols }) => cols); + const keyBatches = generated.map(({ keys }) => keys); + const rows = memoize(() => rowBatches.reduce((rows: any[][], batch) => [...rows, ...batch()], [])); + const keys = memoize(() => keyBatches.reduce((keys: any[][], batch) => ( + !keys.length ? batch() : keys.map((idxs, i) => [...(idxs || []), ...(batch()[i] || [])]) + ), [])); + const cols = memoize(() => colBatches.reduce((cols: any[][], batch) => ( + !cols.length ? batch() : cols.map((vals, i) => [...vals, ...batch()[i]]) + ), [])); + + return { rows, cols, keys, rowBatches, colBatches, keyBatches, table: new Table(schema, generated.map(({ recordBatch }) => recordBatch)) }; +}; + +export const recordBatch = (length = 100, schema: Schema = new Schema(defaultRecordBatchChildren())): GeneratedRecordBatch => { + + const generated = schema.fields.map((f) => vectorGenerator.visit(f.type, length)); + const vecs = generated.map(({ vector }) => vector); + + const keys = memoize(() => generated.map(({ keys }) => keys)); + const cols = memoize(() => generated.map(({ values }) => values())); + const rows = ((_cols: () => any[][]) => memoize((rows: any[][] = [], cols: any[][] = _cols()) => { + for (let i = -1; ++i < length; rows[i] = cols.map((vals) => vals[i])); + return rows; + }))(cols); + + return { rows, cols, keys, recordBatch: new RecordBatch(schema, length, vecs) }; +}; + +export const null_ = (length = 100) => vectorGenerator.visit(new Null(), length); +export const bool = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new Bool(), length, nullCount); +export const int8 = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new Int8(), length, nullCount); +export const int16 = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new Int16(), length, nullCount); +export const int32 = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new Int32(), length, nullCount); +export const int64 = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new Int64(), length, nullCount); +export const uint8 = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new Uint8(), length, nullCount); +export const uint16 = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new Uint16(), length, nullCount); +export const uint32 = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new Uint32(), length, nullCount); +export const uint64 = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new Uint64(), length, nullCount); +export const float16 = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new Float16(), length, nullCount); +export const float32 = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new Float32(), length, nullCount); +export const float64 = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new Float64(), length, nullCount); +export const utf8 = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new Utf8(), length, nullCount); +export const binary = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new Binary(), length, nullCount); +export const fixedSizeBinary = (length = 100, nullCount = length * 0.2 | 0, byteWidth = 8) => vectorGenerator.visit(new FixedSizeBinary(byteWidth), length, nullCount); +export const dateDay = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new DateDay(), length, nullCount); +export const dateMillisecond = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new DateMillisecond(), length, nullCount); +export const timestampSecond = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new TimestampSecond(), length, nullCount); +export const timestampMillisecond = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new TimestampMillisecond(), length, nullCount); +export const timestampMicrosecond = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new TimestampMicrosecond(), length, nullCount); +export const timestampNanosecond = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new TimestampNanosecond(), length, nullCount); +export const timeSecond = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new TimeSecond(), length, nullCount); +export const timeMillisecond = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new TimeMillisecond(), length, nullCount); +export const timeMicrosecond = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new TimeMicrosecond(), length, nullCount); +export const timeNanosecond = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new TimeNanosecond(), length, nullCount); +export const decimal = (length = 100, nullCount = length * 0.2 | 0, scale = 2, precision = 9) => vectorGenerator.visit(new Decimal(scale, precision), length, nullCount); +export const list = (length = 100, nullCount = length * 0.2 | 0, child = defaultListChild) => vectorGenerator.visit(new List(child), length, nullCount); +export const struct = <T extends { [key: string]: DataType } = any>(length = 100, nullCount = length * 0.2 | 0, children: Field<T[keyof T]>[] = <any> defaultStructChildren()) => vectorGenerator.visit(new Struct<T>(children), length, nullCount); +export const denseUnion = (length = 100, nullCount = length * 0.2 | 0, children: Field[] = defaultUnionChildren()) => vectorGenerator.visit(new DenseUnion(children.map((f) => f.typeId), children), length, nullCount); +export const sparseUnion = (length = 100, nullCount = length * 0.2 | 0, children: Field[] = defaultUnionChildren()) => vectorGenerator.visit(new SparseUnion(children.map((f) => f.typeId), children), length, nullCount); +export const dictionary = <T extends DataType = Utf8, TKey extends TKeys = Int32> (length = 100, nullCount = length * 0.2 | 0, dict: T = <any> new Utf8(), keys: TKey = <any> new Int32()) => vectorGenerator.visit(new Dictionary(dict, keys), length, nullCount); +export const intervalDayTime = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new IntervalDayTime(), length, nullCount); +export const intervalYearMonth = (length = 100, nullCount = length * 0.2 | 0) => vectorGenerator.visit(new IntervalYearMonth(), length, nullCount); +export const fixedSizeList = (length = 100, nullCount = length * 0.2 | 0, listSize = 2, child = defaultListChild) => vectorGenerator.visit(new FixedSizeList(listSize, child), length, nullCount); +export const map = <TKey extends DataType = any, TValue extends DataType = any>(length = 100, nullCount = length * 0.2 | 0, child: Field<Struct<{key: TKey; value: TValue}>> = <any> defaultMapChild()) => vectorGenerator.visit(new Map_<TKey, TValue>(child), length, nullCount); + +export const vecs = { + null_, bool, int8, int16, int32, int64, uint8, uint16, uint32, uint64, float16, float32, float64, utf8, binary, fixedSizeBinary, dateDay, dateMillisecond, timestampSecond, timestampMillisecond, timestampMicrosecond, timestampNanosecond, timeSecond, timeMillisecond, timeMicrosecond, timeNanosecond, decimal, list, struct, denseUnion, sparseUnion, dictionary, intervalDayTime, intervalYearMonth, fixedSizeList, map +} as { [k: string]: (...args: any[]) => any }; + +function generateNull<T extends Null>(this: TestDataVectorGenerator, type: T, length = 100): GeneratedVector<V<T>> { + return { values: () => Array.from({ length }, () => null), vector: Vector.new(Data.Null(type, 0, length)) }; +} + +function generateBool<T extends Bool>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0): GeneratedVector<V<T>> { + const data = createBitmap(length, length / 2 | 0); + const nullBitmap = createBitmap(length, nullCount); + const values = memoize(() => { + const values = [] as (boolean | null)[]; + iterateBitmap(length, nullBitmap, (i, valid) => values[i] = !valid ? null : isValid(data, i)); + return values; + }); + iterateBitmap(length, nullBitmap, (i, valid) => !valid && (data[i >> 3] &= ~(1 << (i % 8)))); + + return { values, vector: Vector.new(Data.Bool(type, 0, length, nullCount, nullBitmap, data)) }; +} + +function generateInt<T extends Int>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0): GeneratedVector<V<T>> { + const ArrayType = type.ArrayType; + const stride = 1 + Number(type.bitWidth > 32); + const nullBitmap = createBitmap(length, nullCount); + const data = fillRandom(ArrayType as any, length * stride); + const values = memoize(() => { + const values = [] as (number | null)[]; + iterateBitmap(length, nullBitmap, (i, valid) => { + values[i] = !valid ? null + : stride === 1 ? data[i] + : data.subarray(i * stride, (i + 1) * stride); + }); + return values; + }); + iterateBitmap(length, nullBitmap, (i, valid) => !valid && (data.set(new Uint8Array(stride), i * stride))); + return { values, vector: Vector.new(Data.Int(type, 0, length, nullCount, nullBitmap, data)) }; +} + +function generateFloat<T extends Float>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0): GeneratedVector<V<T>> { + const ArrayType = type.ArrayType; + const precision = type.precision; + const data = fillRandom(ArrayType as any, length); + const nullBitmap = createBitmap(length, nullCount); + const values = memoize(() => { + const values = [] as (number | null)[]; + iterateBitmap(length, nullBitmap, (i, valid) => { + values[i] = !valid ? null : precision > 0 ? data[i] : util.uint16ToFloat64(data[i]); + }); + return values; + }); + iterateBitmap(length, nullBitmap, (i, valid) => data[i] = !valid ? 0 : data[i] * Math.random()); + return { values, vector: Vector.new(Data.Float(type, 0, length, nullCount, nullBitmap, data)) }; +} + +function generateUtf8<T extends Utf8>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0): GeneratedVector<V<T>> { + const nullBitmap = createBitmap(length, nullCount); + const offsets = createVariableWidthOffsets(length, nullBitmap, undefined, undefined, nullCount != 0); + const values: string[] = new Array(offsets.length - 1).fill(null); + [...offsets.slice(1)] + .map((o, i) => isValid(nullBitmap, i) ? o - offsets[i] : null) + .reduce((map, length, i) => { + if (length !== null) { + if (length > 0) { + do { + values[i] = randomString(length); + } while (map.has(values[i])); + return map.set(values[i], i); + } + values[i] = ''; + } + return map; + }, new Map<string, number>()); + const data = createVariableWidthBytes(length, nullBitmap, offsets, (i) => encodeUtf8(values[i])); + return { values: () => values, vector: Vector.new(Data.Utf8(type, 0, length, nullCount, nullBitmap, offsets, data)) }; +} + +function generateBinary<T extends Binary>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0): GeneratedVector<V<T>> { + const nullBitmap = createBitmap(length, nullCount); + const offsets = createVariableWidthOffsets(length, nullBitmap, undefined, undefined, nullCount != 0); + const values = [...offsets.slice(1)] + .map((o, i) => isValid(nullBitmap, i) ? o - offsets[i] : null) + .map((length) => length == null ? null : randomBytes(length)); + const data = createVariableWidthBytes(length, nullBitmap, offsets, (i) => values[i]!); + return { values: () => values, vector: Vector.new(Data.Binary(type, 0, length, nullCount, nullBitmap, offsets, data)) }; +} + +function generateFixedSizeBinary<T extends FixedSizeBinary>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0): GeneratedVector<V<T>> { + const nullBitmap = createBitmap(length, nullCount); + const data = fillRandom(Uint8Array, length * type.byteWidth); + const values = memoize(() => { + const values = [] as (Uint8Array | null)[]; + iterateBitmap(length, nullBitmap, (i, valid) => { + values[i] = !valid ? null : data.subarray(i * type.byteWidth, (i + 1) * type.byteWidth); + }); + return values; + }); + iterateBitmap(length, nullBitmap, (i, valid) => !valid && data.set(new Uint8Array(type.byteWidth), i * type.byteWidth)); + return { values, vector: Vector.new(Data.FixedSizeBinary(type, 0, length, nullCount, nullBitmap, data)) }; +} + +function generateDate<T extends Date_>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0): GeneratedVector<V<T>> { + const values = [] as (number | null)[]; + const nullBitmap = createBitmap(length, nullCount); + const data = type.unit === DateUnit.DAY + ? createDate32(length, nullBitmap, values) + : createDate64(length, nullBitmap, values); + return { + values: () => values.map((x) => x == null ? null : new Date(x)), + vector: Vector.new(Data.Date(type, 0, length, nullCount, nullBitmap, data)) + }; +} + +function generateTimestamp<T extends Timestamp>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0): GeneratedVector<V<T>> { + const values = [] as (number | null)[]; + const nullBitmap = createBitmap(length, nullCount); + const multiple = type.unit === TimeUnit.NANOSECOND ? 1000000000 : + type.unit === TimeUnit.MICROSECOND ? 1000000 : + type.unit === TimeUnit.MILLISECOND ? 1000 : 1; + const data = createTimestamp(length, nullBitmap, multiple, values); + return { values: () => values, vector: Vector.new(Data.Timestamp(type, 0, length, nullCount, nullBitmap, data)) }; +} + +function generateTime<T extends Time>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0): GeneratedVector<V<T>> { + const values = [] as (Int32Array | number | null)[]; + const nullBitmap = createBitmap(length, nullCount); + const multiple = type.unit === TimeUnit.NANOSECOND ? 1000000000 : + type.unit === TimeUnit.MICROSECOND ? 1000000 : + type.unit === TimeUnit.MILLISECOND ? 1000 : 1; + const data = type.bitWidth === 32 + ? createTime32(length, nullBitmap, multiple, values as (number | null)[]) + : createTime64(length, nullBitmap, multiple, values as (Int32Array | null)[]); + return { values: () => values, vector: Vector.new(Data.Time(type, 0, length, nullCount, nullBitmap, data)) }; +} + +function generateDecimal<T extends Decimal>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0): GeneratedVector<V<T>> { + const data = fillRandom(Uint32Array, length * 4); + const nullBitmap = createBitmap(length, nullCount); + const view = new DataView(data.buffer, 0, data.byteLength); + const values = memoize(() => { + const values = [] as (Uint32Array | null)[]; + iterateBitmap(length, nullBitmap, (i, valid) => { + values[i] = !valid ? null : new Uint32Array(data.buffer, 16 * i, 4); + }); + return values; + }); + iterateBitmap(length, nullBitmap, (i, valid) => { + if (!valid) { + view.setFloat64(4 * (i + 0), 0, true); + view.setFloat64(4 * (i + 1), 0, true); + } + }); + return { values, vector: Vector.new(Data.Decimal(type, 0, length, nullCount, nullBitmap, data))}; +} + +function generateInterval<T extends Interval>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0): GeneratedVector<V<T>> { + const stride = (1 + type.unit); + const nullBitmap = createBitmap(length, nullCount); + const data = fillRandom(Int32Array, length * stride); + const values = memoize(() => { + const values = [] as (Int32Array | null)[]; + iterateBitmap(length, nullBitmap, (i: number, valid: boolean) => { + values[i] = !valid ? null : stride === 2 + ? new Int32Array(data.buffer, 4 * i * stride, stride) + : new Int32Array([data[i] / 12 | 0, data[i] % 12 | 0]); + }); + return values; + }); + iterateBitmap(length, nullBitmap, (i: number, valid: boolean) => { + !valid && data.set(new Int32Array(stride), i * stride); + }); + return { values, vector: Vector.new(Data.Interval(type, 0, length, nullCount, nullBitmap, data)) }; +} + +function generateList<T extends List>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0, child = this.visit(type.children[0].type, length * 3, nullCount * 3)): GeneratedVector<V<T>> { + const childVec = child.vector; + const nullBitmap = createBitmap(length, nullCount); + const stride = childVec.length / (length - nullCount); + const offsets = createVariableWidthOffsets(length, nullBitmap, childVec.length, stride); + const values = memoize(() => { + const childValues = child.values(); + const values: (T['valueType'] | null)[] = [...offsets.slice(1)] + .map((offset, i) => isValid(nullBitmap, i) ? offset : null) + .map((o, i) => o == null ? null : childValues.slice(offsets[i], o)); + return values; + }); + return { values, vector: Vector.new(Data.List(type, 0, length, nullCount, nullBitmap, offsets, childVec)) }; +} + +function generateFixedSizeList<T extends FixedSizeList>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0, child = this.visit(type.children[0].type, length * type.listSize, nullCount * type.listSize)): GeneratedVector<V<T>> { + const nullBitmap = createBitmap(length, nullCount); + const values = memoize(() => { + const childValues = child.values(); + const values = [] as (T['valueType'] | null)[]; + for (let i = -1, stride = type.listSize; ++i < length;) { + values[i] = isValid(nullBitmap, i) ? childValues.slice(i * stride, (i + 1) * stride) : null; + } + return values; + }); + return { values, vector: Vector.new(Data.FixedSizeList(type, 0, length, nullCount, nullBitmap, child.vector)) }; +} + +function generateDictionary<T extends Dictionary>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0, dictionary = this.visit(type.dictionary, length, 0)): GeneratedVector<V<T>> { + + const t = <any> type; + const currValues = t.dictionaryValues; + const hasDict = t.dictionaryVector && t.dictionaryVector.length > 0; + const dict = hasDict ? t.dictionaryVector.concat(dictionary.vector) : dictionary.vector; + const vals = hasDict ? (() => [...currValues(), ...dictionary.values()]) : dictionary.values; + + const maxIdx = dict.length - 1; + const keys = new t.indices.ArrayType(length); + const nullBitmap = createBitmap(length, nullCount); + + const values = memoize(() => { + const dict = vals(); + const values = [] as (T['TValue'] | null)[]; + iterateBitmap(length, nullBitmap, (i, valid) => { + values[i] = !valid ? null : dict[keys[i]]; + }); + return values; + }); + + iterateBitmap(length, nullBitmap, (i, valid) => { + keys[i] = !valid ? 0 : rand() * maxIdx | 0; + }); + + t.dictionaryVector = dict; + t.dictionaryValues = vals; + + return { values, keys, vector: Vector.new(Data.Dictionary(type, 0, length, nullCount, nullBitmap, keys, dict)) }; +} + +function generateUnion<T extends Union>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0, children?: GeneratedVector<any>[]): GeneratedVector<V<T>> { + + const numChildren = type.children.length; + + if (!children) { + if (type.mode === UnionMode.Sparse) { + children = type.children.map((f) => this.visit(f.type, length, nullCount)); + } else { + const childLength = Math.ceil(length / numChildren); + const childNullCount = (nullCount / childLength) | 0; + children = type.children.map((f) => this.visit(f.type, childLength, childNullCount)); + } + } + + const typeIds = type.typeIds; + const typeIdsBuffer = new Int8Array(length); + const vecs = children.map(({ vector }) => vector); + const cols = children.map(({ values }) => values); + const nullBitmap = createBitmap(length, nullCount); + const typeIdToChildIndex = typeIds.reduce((typeIdToChildIndex, typeId, idx) => { + return (typeIdToChildIndex[typeId] = idx) && typeIdToChildIndex || typeIdToChildIndex; + }, Object.create(null) as { [key: number]: number }); + + if (type.mode === UnionMode.Sparse) { + const values = memoize(() => { + const values = [] as any[]; + const childValues = cols.map((x) => x()); + iterateBitmap(length, nullBitmap, (i, valid) => { + values[i] = !valid ? null : childValues[typeIdToChildIndex[typeIdsBuffer[i]]][i]; + }); + return values; + }); + iterateBitmap(length, nullBitmap, (i, valid) => { + typeIdsBuffer[i] = !valid ? 0 : typeIds[rand() * numChildren | 0]; + }); + return { values, vector: Vector.new(Data.Union(type as SparseUnion, 0, length, nullCount, nullBitmap, typeIdsBuffer, vecs)) } as GeneratedVector<V<T>>; + } + + const offsets = new Int32Array(length); + const values = memoize(() => { + const values = [] as any[]; + const childValues = cols.map((x) => x()); + iterateBitmap(length, nullBitmap, (i, valid) => { + values[i] = !valid ? null : childValues[typeIdToChildIndex[typeIdsBuffer[i]]][offsets[i]]; + }); + return values; + }); + iterateBitmap(length, nullBitmap, (i, valid) => { + if (!valid) { + offsets[i] = 0; + typeIdsBuffer[i] = 0; + } else { + const colIdx = rand() * numChildren | 0; + offsets[i] = i / numChildren | 0; + typeIdsBuffer[i] = typeIds[colIdx]; + } + }); + return { values, vector: Vector.new(Data.Union(type as DenseUnion, 0, length, nullCount, nullBitmap, typeIdsBuffer, offsets, vecs)) } as GeneratedVector<V<T>>; +} + +function generateStruct<T extends Struct>(this: TestDataVectorGenerator, type: T, length = 100, nullCount = length * 0.2 | 0, children = type.children.map((f) => this.visit(f.type, length, nullCount))): GeneratedVector<V<T>> { + const vecs = children.map(({ vector }) => vector); + const cols = children.map(({ values }) => values); + const nullBitmap = createBitmap(length, nullCount); + const values = memoize(() => { + const values = [] as any[]; + const childValues = cols.map((x) => x()); + const names = type.children.map((f) => f.name); + iterateBitmap(length, nullBitmap, (i, valid) => { + values[i] = !valid ? null : childValues.reduce((row, col, j) => ({ + ...row, [names[j]]: col[i] + }), {}); + }); + return values; + }); + return { values, vector: Vector.new(Data.Struct(type, 0, length, nullCount, nullBitmap, vecs)) }; +} + +function generateMap<T extends Map_>(this: TestDataVectorGenerator, + type: T, length = 100, nullCount = length * 0.2 | 0, + child = this.visit(type.children[0].type, length * 3, 0, [ + this.visit(type.children[0].type.children[0].type, length * 3, 0), + this.visit(type.children[0].type.children[1].type, length * 3, nullCount * 3) + ])): GeneratedVector<V<T>> { + + type K = T['keyType']['TValue']; + type V = T['valueType']['TValue']; + + const childVec = child.vector; + const nullBitmap = createBitmap(length, nullCount); + const stride = childVec.length / (length - nullCount); + const offsets = createVariableWidthOffsets(length, nullBitmap, childVec.length, stride); + const values = memoize(() => { + const childValues = child.values() as { key: K; value: V }[]; + const values: (T['TValue'] | null)[] = [...offsets.slice(1)] + .map((offset, i) => isValid(nullBitmap, i) ? offset : null) + .map((o, i) => o == null ? null : (() => { + const slice = childValues.slice(offsets[i], o); + const pairs = slice.map(({ key, value }) => [key, value]); + return new Map<K, V>(pairs as any as (readonly [K, V])[]); + })()); + return values; + }); + return { values, vector: Vector.new(Data.Map(type, 0, length, nullCount, nullBitmap, offsets, childVec)) }; +} + +type TypedArrayConstructor = + (typeof Int8Array) | + (typeof Int16Array) | + (typeof Int32Array) | + (typeof Uint8Array) | + (typeof Uint16Array) | + (typeof Uint32Array) | + (typeof Float32Array) | + (typeof Float64Array); + + +const rand = Math.random.bind(Math); +const randomBytes = (length: number) => fillRandom(Uint8Array, length); +const randomString = (length: number) => randomatic('?', length, { chars: `abcdefghijklmnopqrstuvwxyz0123456789_` }); + +const memoize = (fn: () => any) => ((x?: any) => () => x || (x = fn()))(); + +const encodeUtf8 = ((encoder) => + encoder.encode.bind(encoder) as (input?: string, options?: { stream?: boolean }) => Uint8Array +)(new TextEncoder()); + +function fillRandom<T extends TypedArrayConstructor>(ArrayType: T, length: number) { + const BPE = ArrayType.BYTES_PER_ELEMENT; + const array = new ArrayType(length); + const max = (2 ** (8 * BPE)) - 1; + for (let i = -1; ++i < length; array[i] = rand() * max * (rand() > 0.5 ? -1 : 1)); + return array as InstanceType<T>; +} + +function isValid(bitmap: Uint8Array, i: number) { + return (bitmap[i >> 3] & 1 << (i % 8)) !== 0; +} + +function iterateBitmap(length: number, bitmap: Uint8Array, fn: (index: number, valid: boolean) => any) { + let byteIndex = 0, valueIndex = 0; + for (let bit = 0; length > 0; bit = 0) { + let byte = bitmap[byteIndex++]; + do { + fn(valueIndex++, (byte & 1 << bit) !== 0); + } while (--length > 0 && ++bit < 8); + } +} + +function createBitmap(length: number, nullCount: number) { + const nulls = Object.create(null) as { [key: number]: boolean }; + const bytes = new Uint8Array((((length >> 3) + 7) & ~7) || 8).fill(255); + for (let i, j = -1; ++j < nullCount;) { + while (nulls[i = (rand() * length) | 0]); + nulls[i] = true; + bytes[i >> 3] &= ~(1 << (i % 8)); // false + } + return bytes; +} + +function createVariableWidthOffsets(length: number, nullBitmap: Uint8Array, max = Infinity, stride = 20, allowEmpty = true) { + const offsets = new Int32Array(length + 1); + iterateBitmap(length, nullBitmap, (i, valid) => { + if (!valid) { + offsets[i + 1] = offsets[i]; + } else { + do { + offsets[i + 1] = Math.min(max, offsets[i] + (rand() * stride | 0)); + } while (!allowEmpty && offsets[i + 1] === offsets[i]); + } + }); + return offsets; +} + +function createVariableWidthBytes(length: number, nullBitmap: Uint8Array, offsets: Int32Array, getBytes: (index: number) => Uint8Array) { + const bytes = new Uint8Array(offsets[length]); + iterateBitmap(length, nullBitmap, (i, valid) => { + valid && bytes.set(getBytes(i), offsets[i]); + }); + return bytes; +} + +function createDate32(length: number, nullBitmap: Uint8Array, values: (number | null)[] = []) { + const data = new Int32Array(length).fill(Date.now() / 86400000 | 0); + iterateBitmap(length, nullBitmap, (i, valid) => { + if (!valid) { + data[i] = 0; + values[i] = null; + } else { + data[i] = data[i] + (rand() * 10000 * (rand() > 0.5 ? -1 : 1)) | 0; + values[i] = data[i] * 86400000; + } + }); + return data; +} + +function createDate64(length: number, nullBitmap: Uint8Array, values: (number | null)[] = []) { + const data = new Int32Array(length * 2).fill(0); + const data32 = createDate32(length, nullBitmap, values); + iterateBitmap(length, nullBitmap, (i, valid) => { + if (valid) { + const value = data32[i] * 86400000; + const hi = (value / 4294967296) | 0; + const lo = (value - 4294967296 * hi) | 0; + values[i] = value; + data[i * 2 + 0] = lo; + data[i * 2 + 1] = hi; + } + }); + return data; +} + +function createTimestamp(length: number, nullBitmap: Uint8Array, multiple: number, values: (number | null)[] = []) { + const mult = 86400 * multiple; + const data = new Int32Array(length * 2).fill(0); + const data32 = createDate32(length, nullBitmap, values); + iterateBitmap(length, nullBitmap, (i, valid) => { + if (valid) { + const value = data32[i] * mult; + const hi = (value / 4294967296) | 0; + const lo = (value - 4294967296 * hi) | 0; + data[i * 2 + 0] = lo; + data[i * 2 + 1] = hi; + } + }); + return data; +} + +function createTime32(length: number, nullBitmap: Uint8Array, multiple: number, values: (number | null)[] = []) { + const data = new Int32Array(length).fill(0); + iterateBitmap(length, nullBitmap, (i, valid) => { + if (!valid) { + data[i] = 0; + values[i] = null; + } else { + values[i] = data[i] = ((1000 * rand()) | 0 * multiple) * (rand() > 0.5 ? -1 : 1); + } + }); + return data; +} + +function createTime64(length: number, nullBitmap: Uint8Array, multiple: number, values: (Int32Array | null)[] = []) { + const data = new Int32Array(length * 2).fill(0); + iterateBitmap(length, nullBitmap, (i, valid) => { + if (!valid) { + values[i] = null; + } else { + const value = (1000 * rand()) | 0 * multiple; + const hi = (value / 4294967296) | 0; + const lo = (value - 4294967296 * hi) | 0; + data[i * 2 + 0] = lo; + data[i * 2 + 1] = hi; + values[i] = data.subarray(i * 2, (i + 1) * 2); + } + }); + return data; +} diff --git a/src/arrow/js/test/inference/column.ts b/src/arrow/js/test/inference/column.ts new file mode 100644 index 000000000..440116b69 --- /dev/null +++ b/src/arrow/js/test/inference/column.ts @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* eslint-disable jest/no-standalone-expect */ + +import { Data } from 'apache-arrow/data'; +import { Field } from 'apache-arrow/schema'; +import { Column } from 'apache-arrow/column'; +import { Vector } from 'apache-arrow/vector'; +import { Bool, Int8, Utf8, List, Dictionary, Struct } from 'apache-arrow/type'; + +const boolType = new Bool(); +const boolVector = Vector.new(Data.Bool(boolType, 0, 10, 0, null, new Uint8Array(2))); + +const boolColumn = new Column(new Field('bool', boolType), [ + Vector.new(Data.Bool(boolType, 0, 10, 0, null, new Uint8Array(2))), + Vector.new(Data.Bool(boolType, 0, 10, 0, null, new Uint8Array(2))), + Vector.new(Data.Bool(boolType, 0, 10, 0, null, new Uint8Array(2))), +]); + +expect(typeof boolVector.get(0) === 'boolean').toBe(true); +expect(typeof boolColumn.get(0) === 'boolean').toBe(true); + +type IndexSchema = { + 0: Int8; + 1: Utf8; + 2: Dictionary<List<Bool>>; +}; + +const structChildFields = [ + { name: 0, type: new Int8() }, + { name: 1, type: new Utf8() }, + { name: 2, type: new Dictionary<List<Bool>>(null!, null!) } +].map(({ name, type }) => new Field('' + name, type)); + +const structType = new Struct<IndexSchema>(structChildFields); +const structVector = Vector.new(Data.Struct(structType, 0, 0, 0, null, [])); +const structColumn = new Column(new Field('struct', structType), [ + Vector.new(Data.Struct(structType, 0, 0, 0, null, [])), + Vector.new(Data.Struct(structType, 0, 0, 0, null, [])), + Vector.new(Data.Struct(structType, 0, 0, 0, null, [])), +]); + +const [x1, y1, z1] = structVector.get(0)!; +const [x2, y2, z2] = structColumn.get(0)!; + +console.log(x1, y1, z1); +console.log(x2, y2, z2); diff --git a/src/arrow/js/test/inference/nested.ts b/src/arrow/js/test/inference/nested.ts new file mode 100644 index 000000000..0e3dc95e3 --- /dev/null +++ b/src/arrow/js/test/inference/nested.ts @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data } from 'apache-arrow/data'; +import { Field } from 'apache-arrow/schema'; +import { DataType } from 'apache-arrow/type'; +import { Vector, BoolVector } from 'apache-arrow/vector/index'; +import { Bool, Int8, Utf8, List, Dictionary, Struct } from 'apache-arrow/type'; + +type NamedSchema = { a: Int8; b: Utf8; c: Dictionary<List<Bool>>; [idx: string]: DataType }; +type IndexSchema = { 0: Int8; 1: Utf8; 2: Dictionary<List<Bool>>; [idx: number]: DataType }; + +checkIndexTypes({ 0: new Int8(), 1: new Utf8(), 2: new Dictionary<List<Bool>>(null!, null!) } as IndexSchema); +checkNamedTypes({ a: new Int8(), b: new Utf8(), c: new Dictionary<List<Bool>>(null!, null!) } as NamedSchema); + +function checkIndexTypes(schema: IndexSchema) { + + const data = Data.Struct(new Struct<IndexSchema>( + Object.keys(schema).map((x) => new Field(x, schema[(<any> x)])) + ), 0, 0, 0, null, []); + + const row = Vector.new(data).bind(0); + + const check_0 = (x = row[0]) => expect(typeof x === 'number').toBe(true); + const check_1 = (x = row[1]) => expect(typeof x === 'string').toBe(true); + const check_2 = (x = row[2]) => expect(x instanceof BoolVector).toBe(true); + + check_0(); check_0(row[0]); check_0(row.get(0)); + check_1(); check_1(row[1]); check_1(row.get(1)); + check_2(); check_2(row[2]); check_2(row.get(2)); +} + +function checkNamedTypes(schema: NamedSchema) { + + const data = Data.Struct(new Struct<NamedSchema>( + Object.keys(schema).map((x) => new Field(x, schema[x])) + ), 0, 0, 0, null, []); + + const row = Vector.new(data).bind(0); + + const check_a = (x = row.a) => expect(typeof x === 'number').toBe(true); + const check_b = (x = row.b) => expect(typeof x === 'string').toBe(true); + const check_c = (x = row.c) => expect(x instanceof BoolVector).toBe(true); + + check_a(); check_a(row.a); check_a(row.get('a')); + check_b(); check_b(row.b); check_b(row.get('b')); + check_c(); check_c(row.c); check_c(row.get('c')); +} diff --git a/src/arrow/js/test/inference/visitor/get.ts b/src/arrow/js/test/inference/visitor/get.ts new file mode 100644 index 000000000..a983d94d1 --- /dev/null +++ b/src/arrow/js/test/inference/visitor/get.ts @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { + Data, Vector, + Bool, List, Dictionary +} from '../../Arrow'; + +import { instance as getVisitor } from 'apache-arrow/visitor/get'; + +const data_Bool = new Data(new Bool(), 0, 0); +const data_List_Bool = new Data(new List<Bool>(null as any), 0, 0); +const data_Dictionary_Bool = new Data(new Dictionary<Bool>(null!, null!), 0, 0); +const data_Dictionary_List_Bool = new Data(new Dictionary<List<Bool>>(null!, null!), 0, 0); + +const boolVec = Vector.new(data_Bool); +const boolVec_getRaw = boolVec.get(0); +const boolVec_getVisit = getVisitor.visit(boolVec, 0); +const boolVec_getFactory = getVisitor.getVisitFn(boolVec)(boolVec, 0); +const boolVec_getFactoryData = getVisitor.getVisitFn(boolVec.data)(boolVec, 0); +const boolVec_getFactoryType = getVisitor.getVisitFn(boolVec.type)(boolVec, 0); + +const listVec = Vector.new(data_List_Bool); +const listVec_getRaw = listVec.get(0); +const listVec_getVisit = getVisitor.visit(listVec, 0); +const listVec_getFactory = getVisitor.getVisitFn(listVec)(listVec, 0); +const listVec_getFactoryData = getVisitor.getVisitFn(listVec.data)(listVec, 0); +const listVec_getFactoryType = getVisitor.getVisitFn(listVec.type)(listVec, 0); + +const dictVec = Vector.new(data_Dictionary_Bool); +const dictVec_getRaw = dictVec.get(0); +const dictVec_getVisit = getVisitor.visit(dictVec, 0); +const dictVec_getFactory = getVisitor.getVisitFn(dictVec)(dictVec, 0); +const dictVec_getFactoryData = getVisitor.getVisitFn(dictVec.data)(dictVec, 0); +const dictVec_getFactoryType = getVisitor.getVisitFn(dictVec.type)(dictVec, 0); + +const dictOfListVec = Vector.new(data_Dictionary_List_Bool); +const dictOfListVec_getRaw = dictOfListVec.get(0); +const dictOfListVec_getVisit = getVisitor.visit(dictOfListVec, 0); +const dictOfListVec_getFactory = getVisitor.getVisitFn(dictOfListVec)(dictOfListVec, 0); +const dictOfListVec_getFactoryData = getVisitor.getVisitFn(dictOfListVec.data)(dictOfListVec, 0); +const dictOfListVec_getFactoryType = getVisitor.getVisitFn(dictOfListVec.type)(dictOfListVec, 0); diff --git a/src/arrow/js/test/jest-extensions.ts b/src/arrow/js/test/jest-extensions.ts new file mode 100644 index 000000000..6adde0b83 --- /dev/null +++ b/src/arrow/js/test/jest-extensions.ts @@ -0,0 +1,162 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { zip } from 'ix/iterable/zip'; +import { Table, Vector, RecordBatch, Column, util } from './Arrow'; + +declare global { + namespace jest { + interface Matchers<R> { + toArrowCompare(expected: any): CustomMatcherResult; + toEqualTable(expected: Table): CustomMatcherResult; + toEqualRecordBatch(expected: RecordBatch): CustomMatcherResult; + toEqualVector(expected: Vector | [Vector | null, string?, string?]): CustomMatcherResult; + } + } +} + +expect.extend({ + toEqualTable, + toEqualVector, + toArrowCompare, + toEqualRecordBatch +}); + +function format(jest: jest.MatcherUtils, actual: any, expected: any, msg= ' ') { + return `${ + jest.utils.printReceived(actual) + }${msg}${ + jest.utils.printExpected(expected) + }`; +} + +function toArrowCompare(this: jest.MatcherUtils, actual: any, expected: any) { + if (!util.createElementComparator(expected)(actual)) { + return { pass: false, message: () => format(this, actual, expected, ' should equal ') }; + } + return { pass: true, message: () => '' }; +} + +function toEqualTable(this: jest.MatcherUtils, actual: Table, expected: Table) { + const failures = [] as string[]; + try { expect(actual).toHaveLength(expected.length); } catch (e) { failures.push(`${e}`); } + try { expect(actual.numCols).toEqual(expected.numCols); } catch (e) { failures.push(`${e}`); } + try { expect(actual.schema.metadata).toEqual(expected.schema.metadata); } catch (e) { failures.push(`${e}`); } + (() => { + for (let i = -1, n = actual.numCols; ++i < n;) { + const v1 = actual.getColumnAt(i); + const v2 = expected.getColumnAt(i); + const name = actual.schema.fields[i].name; + try { + expect([v1, `actual`, name]).toEqualVector([v2, `expected`, name]); + } catch (e) { failures.push(`${e}`); } + } + })(); + return { + pass: failures.length === 0, + message: () => failures.join('\n'), + }; +} + +function toEqualRecordBatch(this: jest.MatcherUtils, actual: RecordBatch, expected: RecordBatch) { + const failures = [] as string[]; + try { expect(actual).toHaveLength(expected.length); } catch (e) { failures.push(`${e}`); } + try { expect(actual.numCols).toEqual(expected.numCols); } catch (e) { failures.push(`${e}`); } + (() => { + for (let i = -1, n = actual.numCols; ++i < n;) { + const v1 = actual.getChildAt(i); + const v2 = expected.getChildAt(i); + const name = actual.schema.fields[i].name; + try { + expect([v1, `actual`, name]).toEqualVector([v2, `expected`, name]); + } catch (e) { failures.push(`${e}`); } + } + })(); + return { + pass: failures.length === 0, + message: () => failures.join('\n'), + }; +} + +function toEqualVector< + TActual extends Vector | [Vector | null, string?, string?], + TExpected extends Vector | [Vector | null, string?] +>(this: jest.MatcherUtils, actual: TActual, expected: TExpected) { + + let [v1, format1 = '', columnName = ''] = Array.isArray(actual) ? actual : [actual]; + let [v2, format2 = ''] = Array.isArray(expected) ? expected : [expected]; + + if (v1 instanceof Column && columnName === '') { columnName = v1.name; } + + if (v1 == null || v2 == null) { + return { + pass: false, + message: () => [ + [columnName, `(${format(this, format1, format2, ' !== ')})`].filter(Boolean).join(':'), + `${v1 == null ? 'actual' : 'expected'} is null` + ].join('\n') + }; + } + + let getFailures = new Array<string>(); + let propsFailures = new Array<string>(); + let iteratorFailures = new Array<string>(); + let allFailures = [ + { title: 'get', failures: getFailures }, + { title: 'props', failures: propsFailures }, + { title: 'iterator', failures: iteratorFailures } + ]; + + let props: (keyof Vector)[] = ['type', 'length', 'nullCount']; + + (() => { + for (let i = -1, n = props.length; ++i < n;) { + const prop = props[i]; + if (`${v1[prop]}` !== `${v2[prop]}`) { + propsFailures.push(`${prop}: ${format(this, v1[prop], v2[prop], ' !== ')}`); + } + } + })(); + + (() => { + for (let i = -1, n = v1.length; ++i < n;) { + let x1 = v1.get(i), x2 = v2.get(i); + if (!util.createElementComparator(x2)(x1)) { + getFailures.push(`${i}: ${format(this, x1, x2, ' !== ')}`); + } + } + })(); + + (() => { + let i = -1; + for (let [x1, x2] of zip(v1, v2)) { + ++i; + if (!util.createElementComparator(x2)(x1)) { + iteratorFailures.push(`${i}: ${format(this, x1, x2, ' !== ')}`); + } + } + })(); + + return { + pass: allFailures.every(({ failures }) => failures.length === 0), + message: () => [ + [columnName, `(${format(this, format1, format2, ' !== ')})`].filter(Boolean).join(':'), + ...allFailures.map(({ failures, title }) => + !failures.length ? `` : [`${title}:`, ...failures].join(`\n`)) + ].join('\n') + }; +} diff --git a/src/arrow/js/test/tsconfig.json b/src/arrow/js/test/tsconfig.json new file mode 100644 index 000000000..8cf2e7e7b --- /dev/null +++ b/src/arrow/js/test/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "../tsconfig.json", + "include": ["../src/**/*.ts", "../test/**/*.ts"], + "compilerOptions": { + "target": "esnext", + "module": "es2020", + "allowJs": true, + "declaration": false, + "declarationMap": false, + "importHelpers": false, + "noEmit": true, + "noEmitHelpers": false, + "noEmitOnError": false, + "sourceMap": true, + "inlineSources": false, + "inlineSourceMap": false, + "downlevelIteration": false, + "baseUrl": "../", + "paths": { + "apache-arrow": ["src/Arrow.node"], + "apache-arrow/*": ["src/*"] + } + } +} diff --git a/src/arrow/js/test/tsconfig/tsconfig.apache-arrow.json b/src/arrow/js/test/tsconfig/tsconfig.apache-arrow.json new file mode 100644 index 000000000..161374e02 --- /dev/null +++ b/src/arrow/js/test/tsconfig/tsconfig.apache-arrow.json @@ -0,0 +1,8 @@ +// TypeScript configuration for the apache-arrow target's tests +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "commonjs" + } +} diff --git a/src/arrow/js/test/tsconfig/tsconfig.base.json b/src/arrow/js/test/tsconfig/tsconfig.base.json new file mode 100644 index 000000000..fcae71fb4 --- /dev/null +++ b/src/arrow/js/test/tsconfig/tsconfig.base.json @@ -0,0 +1,26 @@ +// Base TypeScript configuration for all targets' tests +{ + "extends": "../../tsconfig/tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "commonjs", + "allowJs": true, + "declaration": false, + "importHelpers": false, + "noEmit": false, + "noEmitHelpers": false, + "noEmitOnError": false, + "sourceMap": true, + "inlineSources": false, + "inlineSourceMap": false, + "downlevelIteration": false, + "esModuleInterop": true, + "baseUrl": "../../", + "paths": { + "apache-arrow": ["src/Arrow.node"], + "apache-arrow/*": ["src/*"] + } + }, + "exclude": ["../../node_modules"], + "include": ["../../src/**/*.ts"] +} diff --git a/src/arrow/js/test/tsconfig/tsconfig.coverage.json b/src/arrow/js/test/tsconfig/tsconfig.coverage.json new file mode 100644 index 000000000..e903aa1e5 --- /dev/null +++ b/src/arrow/js/test/tsconfig/tsconfig.coverage.json @@ -0,0 +1,6 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "target": "esnext" + } +} diff --git a/src/arrow/js/test/tsconfig/tsconfig.es2015.cjs.json b/src/arrow/js/test/tsconfig/tsconfig.es2015.cjs.json new file mode 100644 index 000000000..ed600bc24 --- /dev/null +++ b/src/arrow/js/test/tsconfig/tsconfig.es2015.cjs.json @@ -0,0 +1,8 @@ +// TypeScript configuration for the ES2015 CommonJS target's tests +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "commonjs" + } +} diff --git a/src/arrow/js/test/tsconfig/tsconfig.es2015.esm.json b/src/arrow/js/test/tsconfig/tsconfig.es2015.esm.json new file mode 100644 index 000000000..a030beba7 --- /dev/null +++ b/src/arrow/js/test/tsconfig/tsconfig.es2015.esm.json @@ -0,0 +1,8 @@ +// TypeScript configuration for the ES2015 ESModules target's tests +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "es2020" + } +} diff --git a/src/arrow/js/test/tsconfig/tsconfig.es2015.umd.json b/src/arrow/js/test/tsconfig/tsconfig.es2015.umd.json new file mode 100644 index 000000000..3e4de6f3c --- /dev/null +++ b/src/arrow/js/test/tsconfig/tsconfig.es2015.umd.json @@ -0,0 +1,11 @@ +// TypeScript configuration for the ES2015 Closure Compiler target's tests +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "umd", + "declaration": false, + "noEmitHelpers": true, + "importHelpers": true + } +} diff --git a/src/arrow/js/test/tsconfig/tsconfig.es5.cjs.json b/src/arrow/js/test/tsconfig/tsconfig.es5.cjs.json new file mode 100644 index 000000000..edcd69773 --- /dev/null +++ b/src/arrow/js/test/tsconfig/tsconfig.es5.cjs.json @@ -0,0 +1,9 @@ +// TypeScript configuration for the ES5 CommonJS target's tests +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "commonjs", + "downlevelIteration": true + } +} diff --git a/src/arrow/js/test/tsconfig/tsconfig.es5.esm.json b/src/arrow/js/test/tsconfig/tsconfig.es5.esm.json new file mode 100644 index 000000000..01af8fabd --- /dev/null +++ b/src/arrow/js/test/tsconfig/tsconfig.es5.esm.json @@ -0,0 +1,9 @@ +// TypeScript configuration for the ES5 ESModules target's tests +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "es2020", + "downlevelIteration": true + } +} diff --git a/src/arrow/js/test/tsconfig/tsconfig.es5.umd.json b/src/arrow/js/test/tsconfig/tsconfig.es5.umd.json new file mode 100644 index 000000000..445ec8809 --- /dev/null +++ b/src/arrow/js/test/tsconfig/tsconfig.es5.umd.json @@ -0,0 +1,12 @@ +// TypeScript configuration for the ES5 Closure Compiler target's tests +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "umd", + "declaration": false, + "noEmitHelpers": true, + "importHelpers": true, + "downlevelIteration": true + } +} diff --git a/src/arrow/js/test/tsconfig/tsconfig.esnext.cjs.json b/src/arrow/js/test/tsconfig/tsconfig.esnext.cjs.json new file mode 100644 index 000000000..6f21fd56c --- /dev/null +++ b/src/arrow/js/test/tsconfig/tsconfig.esnext.cjs.json @@ -0,0 +1,8 @@ +// TypeScript configuration for the ESNext CommonJS target's tests +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "commonjs" + } +} diff --git a/src/arrow/js/test/tsconfig/tsconfig.esnext.esm.json b/src/arrow/js/test/tsconfig/tsconfig.esnext.esm.json new file mode 100644 index 000000000..3a9c27745 --- /dev/null +++ b/src/arrow/js/test/tsconfig/tsconfig.esnext.esm.json @@ -0,0 +1,8 @@ +// TypeScript configuration for the ESNext ESModules target's tests +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "es2020" + } +} diff --git a/src/arrow/js/test/tsconfig/tsconfig.esnext.umd.json b/src/arrow/js/test/tsconfig/tsconfig.esnext.umd.json new file mode 100644 index 000000000..baccc6994 --- /dev/null +++ b/src/arrow/js/test/tsconfig/tsconfig.esnext.umd.json @@ -0,0 +1,11 @@ +// TypeScript configuration for the ESNext Closure Compiler target's tests +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "umd", + "declaration": false, + "noEmitHelpers": true, + "importHelpers": true + } +} diff --git a/src/arrow/js/test/tsconfig/tsconfig.src.json b/src/arrow/js/test/tsconfig/tsconfig.src.json new file mode 100644 index 000000000..5413898f7 --- /dev/null +++ b/src/arrow/js/test/tsconfig/tsconfig.src.json @@ -0,0 +1,8 @@ +// TypeScript configuration for the source target's tests +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "es2020" + } +} diff --git a/src/arrow/js/test/tsconfig/tsconfig.ts.json b/src/arrow/js/test/tsconfig/tsconfig.ts.json new file mode 100644 index 000000000..1e053698e --- /dev/null +++ b/src/arrow/js/test/tsconfig/tsconfig.ts.json @@ -0,0 +1,8 @@ +// TypeScript configuration for the TypeScript target's tests +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "es2020" + } +} diff --git a/src/arrow/js/test/unit/bit-tests.ts b/src/arrow/js/test/unit/bit-tests.ts new file mode 100644 index 000000000..cdfb37c16 --- /dev/null +++ b/src/arrow/js/test/unit/bit-tests.ts @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import * as Arrow from 'apache-arrow'; +const { BitIterator, getBool } = Arrow.util; + +describe('Bits', () => { + test('BitIterator produces correct bits for single byte', () => { + const byte = new Uint8Array([0b11110000]); + expect([...new BitIterator(byte, 0, 8, null, getBool)]).toEqual( + [false, false, false, false, true, true, true, true]); + + expect([...new BitIterator(byte, 2, 5, null, getBool)]).toEqual( + [false, false, true, true, true]); + }); + + test('BitIterator produces correct bits for multiple bytes', () => { + const byte = new Uint8Array([0b11110000, 0b10101010]); + expect([...new BitIterator(byte, 0, 16, null, getBool)]).toEqual( + [false, false, false, false, true, true, true, true, + false, true, false, true, false, true, false, true]); + + expect([...new BitIterator(byte, 2, 11, null, getBool)]).toEqual( + [false, false, true, true, true, true, + false, true, false, true, false]); + }); +}); diff --git a/src/arrow/js/test/unit/builders/builder-tests.ts b/src/arrow/js/test/unit/builders/builder-tests.ts new file mode 100644 index 000000000..b6fa60271 --- /dev/null +++ b/src/arrow/js/test/unit/builders/builder-tests.ts @@ -0,0 +1,268 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import '../../jest-extensions'; +import { from, fromDOMStream, toArray } from 'ix/asynciterable'; +import { fromNodeStream } from 'ix/asynciterable/fromnodestream'; +import { validateVector } from './utils'; +import * as generate from '../../generate-test-data'; +import { Type, DataType, Chunked, util, Builder, UnionVector } from 'apache-arrow'; + +const testDOMStreams = process.env.TEST_DOM_STREAMS === 'true'; +const testNodeStreams = process.env.TEST_NODE_STREAMS === 'true'; + +describe('Generated Test Data', () => { + describe('NullBuilder', () => { validateBuilder(generate.null_); }); + describe('BoolBuilder', () => { validateBuilder(generate.bool); }); + describe('Int8Builder', () => { validateBuilder(generate.int8); }); + describe('Int16Builder', () => { validateBuilder(generate.int16); }); + describe('Int32Builder', () => { validateBuilder(generate.int32); }); + describe('Int64Builder', () => { validateBuilder(generate.int64); }); + describe('Uint8Builder', () => { validateBuilder(generate.uint8); }); + describe('Uint16Builder', () => { validateBuilder(generate.uint16); }); + describe('Uint32Builder', () => { validateBuilder(generate.uint32); }); + describe('Uint64Builder', () => { validateBuilder(generate.uint64); }); + describe('Float16Builder', () => { validateBuilder(generate.float16); }); + describe('Float32Builder', () => { validateBuilder(generate.float32); }); + describe('Float64Builder', () => { validateBuilder(generate.float64); }); + describe('Utf8Builder', () => { validateBuilder(generate.utf8); }); + describe('BinaryBuilder', () => { validateBuilder(generate.binary); }); + describe('FixedSizeBinaryBuilder', () => { validateBuilder(generate.fixedSizeBinary); }); + describe('DateDayBuilder', () => { validateBuilder(generate.dateDay); }); + describe('DateMillisecondBuilder', () => { validateBuilder(generate.dateMillisecond); }); + describe('TimestampSecondBuilder', () => { validateBuilder(generate.timestampSecond); }); + describe('TimestampMillisecondBuilder', () => { validateBuilder(generate.timestampMillisecond); }); + describe('TimestampMicrosecondBuilder', () => { validateBuilder(generate.timestampMicrosecond); }); + describe('TimestampNanosecondBuilder', () => { validateBuilder(generate.timestampNanosecond); }); + describe('TimeSecondBuilder', () => { validateBuilder(generate.timeSecond); }); + describe('TimeMillisecondBuilder', () => { validateBuilder(generate.timeMillisecond); }); + describe('TimeMicrosecondBuilder', () => { validateBuilder(generate.timeMicrosecond); }); + describe('TimeNanosecondBuilder', () => { validateBuilder(generate.timeNanosecond); }); + describe('DecimalBuilder', () => { validateBuilder(generate.decimal); }); + describe('ListBuilder', () => { validateBuilder(generate.list); }); + describe('StructBuilder', () => { validateBuilder(generate.struct); }); + describe('DenseUnionBuilder', () => { validateBuilder(generate.denseUnion); }); + describe('SparseUnionBuilder', () => { validateBuilder(generate.sparseUnion); }); + describe('DictionaryBuilder', () => { validateBuilder(generate.dictionary); }); + describe('IntervalDayTimeBuilder', () => { validateBuilder(generate.intervalDayTime); }); + describe('IntervalYearMonthBuilder', () => { validateBuilder(generate.intervalYearMonth); }); + describe('FixedSizeListBuilder', () => { validateBuilder(generate.fixedSizeList); }); + describe('MapBuilder', () => { validateBuilder(generate.map); }); +}); + +function validateBuilder(generate: (length?: number, nullCount?: number, ...args: any[]) => generate.GeneratedVector) { + + const type = generate(0, 0).vector.type; + + for (let i = -1; ++i < 1;) { + validateBuilderWithNullValues(`no nulls`, [], generate(100, 0)); + validateBuilderWithNullValues(`with nulls`, [null], generate(100)); + if (DataType.isUtf8(type)) { + validateBuilderWithNullValues(`with \\0`, ['\0'], generate(100)); + validateBuilderWithNullValues(`with n/a`, ['n/a'], generate(100)); + } else if (DataType.isFloat(type)) { + validateBuilderWithNullValues(`with NaNs`, [NaN], generate(100)); + } else if (DataType.isInt(type)) { + validateBuilderWithNullValues(`with MAX_INT`, [ + type.bitWidth < 64 ? 0x7fffffff : + new Uint32Array([0x7fffffff, 0x7fffffff])], generate(100)); + } + } +} + +const countQueueingStrategy = { highWaterMark: 10 }; +const byteLengthQueueingStrategy = { highWaterMark: 64 }; + +const iterableBuilderOptions = <T extends DataType = any>({ vector }: generate.GeneratedVector, { type, ...opts }: BuilderOptions<T>) => ({ + ...opts, type, + valueToChildTypeId: !DataType.isUnion(type) ? undefined : (() => { + let { typeIds } = vector as UnionVector; + let lastChunkLength = 0, chunksLength = 0; + return (builder: Builder<T>, _value: any, index: number) => { + if (index === 0) { + chunksLength += lastChunkLength; + } + lastChunkLength = builder.length + 1; + return typeIds[chunksLength + index]; + }; + })() +}); + +const domStreamBuilderOptions = <T extends DataType = any>({ vector }: generate.GeneratedVector, { type, queueingStrategy, ...opts }: Partial<BuilderTransformOptions<T>>) => ({ + ...opts, type, + valueToChildTypeId: !DataType.isUnion(type) ? undefined : (() => { + let { typeIds } = vector as UnionVector; + let lastChunkLength = 0, chunksLength = 0; + return (builder: Builder<T>, _value: any, index: number) => { + if (index === 0) { + chunksLength += lastChunkLength; + } + lastChunkLength = builder.length + 1; + return typeIds[chunksLength + index]; + }; + })(), + queueingStrategy, + readableStrategy: queueingStrategy === 'bytes' ? byteLengthQueueingStrategy : countQueueingStrategy, + writableStrategy: queueingStrategy === 'bytes' ? byteLengthQueueingStrategy : countQueueingStrategy, +}); + +const nodeStreamBuilderOptions = <T extends DataType = any>({ vector }: generate.GeneratedVector, { type, queueingStrategy, ...opts }: Partial<BuilderDuplexOptions<T>>) => ({ + ...opts, type, + valueToChildTypeId: !DataType.isUnion(type) ? undefined : (() => { + let { typeIds } = vector as UnionVector; + let lastChunkLength = 0, chunksLength = 0; + return (builder: Builder<T>, _value: any, index: number) => { + if (index === 0) { + chunksLength += lastChunkLength; + } + lastChunkLength = builder.length + 1; + return typeIds[chunksLength + index]; + }; + })(), + queueingStrategy, + highWaterMark: queueingStrategy === 'bytes' ? 64 : 10 +}); + +function validateBuilderWithNullValues(suiteName: string, nullValues: any[], generated: generate.GeneratedVector) { + + const type = generated.vector.type; + const referenceNullValues = nullValues.slice(); + const originalValues = generated.values().slice(); + const typeName = Type[type.typeId].toLowerCase(); + + let values: any[]; + const opts: any = { type, nullValues }; + + if (DataType.isNull(type) || (nullValues.length === 1 && nullValues[0] === null)) { + values = originalValues.slice(); + } else if (nullValues.length > 0) { + values = fillNA(originalValues, nullValues); + } else { + values = fillNADefault(originalValues, [originalValues.find((x) => x !== null)]); + } + + if (DataType.isInt(type) && type.bitWidth === 64 && ArrayBuffer.isView(nullValues[0])) { + referenceNullValues[0] = util.BN.new<any>(nullValues[0])[Symbol.toPrimitive]('default'); + } + + describe(suiteName, () => { + it(`encodes ${typeName} single`, async () => { + const opts_ = iterableBuilderOptions(generated, { ...opts }); + const vector = await encodeSingle(values.slice(), opts_); + validateVector(values, vector, referenceNullValues); + }); + it(`encodes ${typeName} chunks by count`, async () => { + const highWaterMark = Math.max(5, (Math.random() * values.length - 5) | 0); + const opts_ = iterableBuilderOptions(generated, { ...opts, highWaterMark, queueingStrategy: 'count' }); + const vector = await encodeChunks(values.slice(), opts_); + validateVector(values, vector, referenceNullValues); + }); + it(`encodes ${typeName} chunks by bytes`, async () => { + const highWaterMark = 64; + const opts_ = iterableBuilderOptions(generated, { ...opts, highWaterMark, queueingStrategy: 'bytes' }); + const vector = await encodeChunks(values.slice(), opts_); + validateVector(values, vector, referenceNullValues); + }); + if (testDOMStreams) { + it(`encodes ${typeName} chunks from a DOM stream by count`, async () => { + const opts_ = domStreamBuilderOptions(generated, { ...opts, queueingStrategy: 'count' }); + const vector = await encodeChunksDOM(values.slice(), opts_); + validateVector(values, vector, referenceNullValues); + }); + it(`encodes ${typeName} chunks from a DOM stream by bytes`, async () => { + const opts_ = domStreamBuilderOptions(generated, { ...opts, queueingStrategy: 'bytes' }); + const vector = await encodeChunksDOM(values.slice(), opts_); + validateVector(values, vector, referenceNullValues); + }); + } + if (testNodeStreams) { + it(`encodes ${typeName} chunks from a Node stream by count`, async () => { + const opts_ = nodeStreamBuilderOptions(generated, { ...opts, queueingStrategy: 'count' }); + const vector = await encodeChunksNode(values.slice(), opts_); + validateVector(values, vector, referenceNullValues); + }); + it(`encodes ${typeName} chunks from a Node stream by bytes`, async () => { + const opts_ = nodeStreamBuilderOptions(generated, { ...opts, queueingStrategy: 'bytes' }); + const vector = await encodeChunksNode(values.slice(), opts_); + validateVector(values, vector, referenceNullValues); + }); + } + }); +} + +function fillNA(values: any[], nulls: any[]): any[] { + const n = nulls.length - 1; + return values.map((x) => { + if (x === null) { + return nulls[Math.round(n * Math.random())]; + } + return x; + }); +} + +function fillNADefault(values: any[], nulls: any[]): any[] { + const n = nulls.length - 1; + return values.map((x) => { + if (x === null) { + return nulls[Math.round(n * Math.random())]; + } else if (Array.isArray(x) && x.length > 0) { + let defaultValue = x.find((y) => y !== null); + if (defaultValue === undefined) { defaultValue = 0; } + return fillNADefault(x, [defaultValue]); + } + return x; + }); +} + +type BuilderOptions<T extends DataType = any, TNull = any> = import('apache-arrow/builder').BuilderOptions<T, TNull>; +type BuilderDuplexOptions<T extends DataType = any, TNull = any> = import('apache-arrow/io/node/builder').BuilderDuplexOptions<T, TNull>; +type BuilderTransformOptions<T extends DataType = any, TNull = any> = import('apache-arrow/io/whatwg/builder').BuilderTransformOptions<T, TNull>; + +async function encodeSingle<T extends DataType, TNull = any>(values: (T['TValue'] | TNull)[], options: BuilderOptions<T, TNull>) { + const builder = Builder.new(options); + values.forEach((x) => builder.append(x)); + return builder.finish().toVector(); +} + +async function encodeChunks<T extends DataType, TNull = any>(values: (T['TValue'] | TNull)[], options: BuilderOptions<T, TNull>) { + return Chunked.concat(...Builder.throughIterable(options)(values)); +} + +async function encodeChunksDOM<T extends DataType, TNull = any>(values: (T['TValue'] | TNull)[], options: BuilderTransformOptions<T, TNull>) { + + const stream = from(values).toDOMStream() + .pipeThrough(Builder.throughDOM(options)); + + const chunks = await fromDOMStream(stream).pipe(toArray); + + return Chunked.concat(...chunks); +} + +async function encodeChunksNode<T extends DataType, TNull = any>(values: (T['TValue'] | TNull)[], options: BuilderDuplexOptions<T, TNull>) { + + if (options.nullValues) { + options.nullValues = [...options.nullValues, undefined] as TNull[]; + } + + const stream = from(fillNA(values, [undefined])) + .toNodeStream({ objectMode: true }) + .pipe(Builder.throughNode(options)); + + const chunks: any[] = await fromNodeStream(stream, options.highWaterMark).pipe(toArray); + + return Chunked.concat(...chunks); +} diff --git a/src/arrow/js/test/unit/builders/date-tests.ts b/src/arrow/js/test/unit/builders/date-tests.ts new file mode 100644 index 000000000..5a9cc092b --- /dev/null +++ b/src/arrow/js/test/unit/builders/date-tests.ts @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { validateVector } from './utils'; +import { Vector, DateDay, DateMillisecond } from 'apache-arrow'; +import { + encodeAll, + encodeEach, + encodeEachDOM, + encodeEachNode, + date32sNoNulls, + date64sNoNulls, + date32sWithNulls, + date64sWithNulls +} from './utils'; + +const testDOMStreams = process.env.TEST_DOM_STREAMS === 'true'; +const testNodeStreams = process.env.TEST_NODE_STREAMS === 'true'; + +describe('DateDayBuilder', () => { + runTestsWithEncoder('encodeAll', encodeAll(() => new DateDay())); + runTestsWithEncoder('encodeEach: 5', encodeEach(() => new DateDay(), 5)); + runTestsWithEncoder('encodeEach: 25', encodeEach(() => new DateDay(), 25)); + runTestsWithEncoder('encodeEach: undefined', encodeEach(() => new DateDay())); + testDOMStreams && runTestsWithEncoder('encodeEachDOM: 25', encodeEachDOM(() => new DateDay(), 25)); + testNodeStreams && runTestsWithEncoder('encodeEachNode: 25', encodeEachNode(() => new DateDay(), 25)); + + function runTestsWithEncoder(name: string, encode: (vals: (Date | null)[], nullVals?: any[]) => Promise<Vector<DateDay>>) { + describe(`${encode.name} ${name}`, () => { + it(`encodes dates no nulls`, async () => { + const vals = date32sNoNulls(20); + validateVector(vals, await encode(vals, []), []); + }); + it(`encodes dates with nulls`, async () => { + const vals = date32sWithNulls(20); + validateVector(vals, await encode(vals, [null]), [null]); + }); + }); + } +}); + +describe('DateMillisecondBuilder', () => { + runTestsWithEncoder('encodeAll', encodeAll(() => new DateMillisecond())); + runTestsWithEncoder('encodeEach: 5', encodeEach(() => new DateMillisecond(), 5)); + runTestsWithEncoder('encodeEach: 25', encodeEach(() => new DateMillisecond(), 25)); + runTestsWithEncoder('encodeEach: undefined', encodeEach(() => new DateMillisecond())); + testDOMStreams && runTestsWithEncoder('encodeEachDOM: 25', encodeEachDOM(() => new DateMillisecond(), 25)); + testNodeStreams && runTestsWithEncoder('encodeEachNode: 25', encodeEachNode(() => new DateMillisecond(), 25)); + + function runTestsWithEncoder(name: string, encode: (vals: (Date | null)[], nullVals?: any[]) => Promise<Vector<DateMillisecond>>) { + describe(`${encode.name} ${name}`, () => { + it(`encodes dates no nulls`, async () => { + const vals = date64sNoNulls(20); + validateVector(vals, await encode(vals, []), []); + }); + it(`encodes dates with nulls`, async () => { + const vals = date64sWithNulls(20); + validateVector(vals, await encode(vals, [null]), [null]); + }); + }); + } +}); + +describe('DateMillisecondBuilder with nulls', () => { + const encode = encodeAll(() => new DateMillisecond()); + const dates = [ + null, + '2019-03-19T13:40:14.746Z', + '2019-03-06T21:12:50.912Z', + '2019-03-22T12:50:56.854Z', + '2019-02-25T03:34:30.916Z', + null, + null, + null, + null, + null, + null, + '2019-03-18T18:12:37.293Z', + '2019-03-26T21:58:35.307Z', + '2019-04-02T03:03:46.464Z', + '2019-03-24T18:45:25.763Z', + null, + '2019-03-19T01:10:59.189Z', + '2019-03-10T21:15:32.237Z', + '2019-03-21T07:25:34.864Z', + null + ].map((x) => x === null ? x : new Date(x)); + it(`encodes dates with nulls`, async () => { + const vals = dates.slice(); + validateVector(vals, await encode(vals, [null]), [null]); + }); +}); diff --git a/src/arrow/js/test/unit/builders/dictionary-tests.ts b/src/arrow/js/test/unit/builders/dictionary-tests.ts new file mode 100644 index 000000000..19b3603bc --- /dev/null +++ b/src/arrow/js/test/unit/builders/dictionary-tests.ts @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { validateVector } from './utils'; +import { Dictionary, Utf8, Int32, Vector } from 'apache-arrow'; +import { + encodeAll, + encodeEach, + encodeEachDOM, + encodeEachNode, + duplicateItems, + stringsNoNulls, + stringsWithNAs, + stringsWithNulls, + stringsWithEmpties +} from './utils'; + +const testDOMStreams = process.env.TEST_DOM_STREAMS === 'true'; +const testNodeStreams = process.env.TEST_NODE_STREAMS === 'true'; + +describe('DictionaryBuilder', () => { + describe('<Utf8, Int32>', () => { + runTestsWithEncoder('encodeAll', encodeAll(() => new Dictionary(new Utf8(), new Int32()))); + runTestsWithEncoder('encodeEach: 5', encodeEach(() => new Dictionary(new Utf8(), new Int32()), 5)); + runTestsWithEncoder('encodeEach: 25', encodeEach(() => new Dictionary(new Utf8(), new Int32()), 25)); + runTestsWithEncoder('encodeEach: undefined', encodeEach(() => new Dictionary(new Utf8(), new Int32()), void 0)); + testDOMStreams && runTestsWithEncoder('encodeEachDOM: 25', encodeEachDOM(() => new Dictionary(new Utf8(), new Int32()), 25)); + testNodeStreams && runTestsWithEncoder('encodeEachNode: 25', encodeEachNode(() => new Dictionary(new Utf8(), new Int32()), 25)); + }); +}); + +function runTestsWithEncoder(name: string, encode: (vals: (string | null)[], nullVals?: any[]) => Promise<Vector<Dictionary<Utf8, Int32>>>) { + describe(`${encode.name} ${name}`, () => { + it(`dictionary-encodes strings no nulls`, async () => { + const vals = duplicateItems(20, stringsNoNulls(10)); + validateVector(vals, await encode(vals, []), []); + }); + it(`dictionary-encodes strings with nulls`, async () => { + const vals = duplicateItems(20, stringsWithNulls(10)); + validateVector(vals, await encode(vals, [null]), [null]); + }); + it(`dictionary-encodes strings using n/a as the null value rep`, async () => { + const vals = duplicateItems(20, stringsWithNAs(10)); + validateVector(vals, await encode(vals, ['n/a']), ['n/a']); + }); + it(`dictionary-encodes strings using \\0 as the null value rep`, async () => { + const vals = duplicateItems(20, stringsWithEmpties(10)); + validateVector(vals, await encode(vals, ['\0']), ['\0']); + }); + }); +} diff --git a/src/arrow/js/test/unit/builders/int64-tests.ts b/src/arrow/js/test/unit/builders/int64-tests.ts new file mode 100644 index 000000000..876ce7030 --- /dev/null +++ b/src/arrow/js/test/unit/builders/int64-tests.ts @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { util, Vector, DataType, Int64 } from 'apache-arrow'; +import { + validateVector, + encodeAll, encodeEach, encodeEachDOM, encodeEachNode, + int64sNoNulls, int64sWithNulls, int64sWithMaxInts, +} from './utils'; + +const testDOMStreams = process.env.TEST_DOM_STREAMS === 'true'; +const testNodeStreams = process.env.TEST_NODE_STREAMS === 'true'; + +const typeFactory = () => new Int64(); +const valueName = `Int64`.toLowerCase(); +const encode0 = encodeAll(typeFactory); +const encode1 = encodeEach(typeFactory); +const encode2 = encodeEach(typeFactory, 5); +const encode3 = encodeEach(typeFactory, 25); +const encode4 = encodeEachDOM(typeFactory, 25); +const encode5 = encodeEachNode(typeFactory, 25); + +const nulls0: any[] = [0x7fffffff]; +const nulls1: any[] = [0x7fffffff]; +nulls0[0] = new Uint32Array([0x7fffffff, 0x7fffffff]); +nulls1[0] = util.BN.new(nulls0[0])[Symbol.toPrimitive](); + +type EncodeValues<T extends DataType> = (values: (T['TValue'] | null)[], nullVals?: any[]) => Promise<Vector<T>>; + +function encodeAndValidate<T extends DataType>(encode: EncodeValues<T>, providedNulls: any[] = [], expectedNulls = providedNulls) { + return (values: any[]) => { + return async () => { + const vector = await encode(values, providedNulls); + const expected = values.map((x) => { + switch (typeof x) { + case 'number': return new Int32Array([x, 0]); + case 'bigint': return new Int32Array(new BigInt64Array([x]).buffer); + } + return x ? x.slice() : x; + }); + return validateVector(expected, vector, expectedNulls); + }; + }; +} + +describe(`Int64Builder`, () => { + describe(`encode single chunk`, () => { + it(`encodes ${valueName} no nulls`, encodeAndValidate(encode0, [], [])(int64sNoNulls(20))); + it(`encodes ${valueName} with nulls`, encodeAndValidate(encode0, [null], [null])(int64sWithNulls(20))); + it(`encodes ${valueName} with MAX_INT`, encodeAndValidate(encode0, nulls0, nulls1)(int64sWithMaxInts(20))); + }); + describe(`encode chunks length default`, () => { + it(`encodes ${valueName} no nulls`, encodeAndValidate(encode1, [], [])(int64sNoNulls(20))); + it(`encodes ${valueName} with nulls`, encodeAndValidate(encode1, [null], [null])(int64sWithNulls(20))); + it(`encodes ${valueName} with MAX_INT`, encodeAndValidate(encode1, nulls0, nulls1)(int64sWithMaxInts(20))); + }); + describe(`encode chunks length 5`, () => { + it(`encodes ${valueName} no nulls`, encodeAndValidate(encode2, [], [])(int64sNoNulls(20))); + it(`encodes ${valueName} with nulls`, encodeAndValidate(encode2, [null], [null])(int64sWithNulls(20))); + it(`encodes ${valueName} with MAX_INT`, encodeAndValidate(encode2, nulls0, nulls1)(int64sWithMaxInts(20))); + }); + describe(`encode chunks length 25`, () => { + it(`encodes ${valueName} no nulls`, encodeAndValidate(encode3, [], [])(int64sNoNulls(20))); + it(`encodes ${valueName} with nulls`, encodeAndValidate(encode3, [null], [null])(int64sWithNulls(20))); + it(`encodes ${valueName} with MAX_INT`, encodeAndValidate(encode3, nulls0, nulls1)(int64sWithMaxInts(20))); + }); + testDOMStreams && describe(`encode chunks length 25, WhatWG stream`, () => { + it(`encodes ${valueName} no nulls`, encodeAndValidate(encode4, [], [])(int64sNoNulls(20))); + it(`encodes ${valueName} with nulls`, encodeAndValidate(encode4, [null], [null])(int64sWithNulls(20))); + it(`encodes ${valueName} with MAX_INT`, encodeAndValidate(encode4, nulls0, nulls1)(int64sWithMaxInts(20))); + }); + testNodeStreams && describe(`encode chunks length 25, NodeJS stream`, () => { + it(`encodes ${valueName} no nulls`, encodeAndValidate(encode5, [], [])(int64sNoNulls(20))); + it(`encodes ${valueName} with nulls`, encodeAndValidate(encode5, [null], [null])(int64sWithNulls(20))); + it(`encodes ${valueName} with MAX_INT`, encodeAndValidate(encode5, nulls0, nulls1)(int64sWithMaxInts(20))); + }); +}); diff --git a/src/arrow/js/test/unit/builders/primitive-tests.ts b/src/arrow/js/test/unit/builders/primitive-tests.ts new file mode 100644 index 000000000..3fd515bf4 --- /dev/null +++ b/src/arrow/js/test/unit/builders/primitive-tests.ts @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { + Vector, DataType, + Bool, Int8, Int16, Int32, Uint8, Uint16, Uint32, Float16, Float32, Float64 +} from 'apache-arrow'; + +import { + validateVector, + encodeAll, encodeEach, encodeEachDOM, encodeEachNode, + boolsNoNulls, boolsWithNulls, + int8sNoNulls, int8sWithNulls, int8sWithMaxInts, + int16sNoNulls, int16sWithNulls, int16sWithMaxInts, + int32sNoNulls, int32sWithNulls, int32sWithMaxInts, + uint8sNoNulls, uint8sWithNulls, uint8sWithMaxInts, + uint16sNoNulls, uint16sWithNulls, uint16sWithMaxInts, + uint32sNoNulls, uint32sWithNulls, uint32sWithMaxInts, + float16sNoNulls, float16sWithNulls, float16sWithNaNs, + float32sNoNulls, float32sWithNulls, float64sWithNaNs, + float64sNoNulls, float64sWithNulls, float32sWithNaNs, +} from './utils'; + +const testDOMStreams = process.env.TEST_DOM_STREAMS === 'true'; +const testNodeStreams = process.env.TEST_NODE_STREAMS === 'true'; + +describe('BoolBuilder', () => { + + runTestsWithEncoder('encodeAll: 5', encodeAll(() => new Bool())); + runTestsWithEncoder('encodeEach: 5', encodeEach(() => new Bool(), 5)); + runTestsWithEncoder('encodeEach: 25', encodeEach(() => new Bool(), 25)); + runTestsWithEncoder('encodeEach: undefined', encodeEach(() => new Bool())); + testDOMStreams && runTestsWithEncoder('encodeEachDOM: 25', encodeEachDOM(() => new Bool(), 25)); + testNodeStreams && runTestsWithEncoder('encodeEachNode: 25', encodeEachNode(() => new Bool(), 25)); + + function runTestsWithEncoder<T extends DataType>(name: string, encode: (vals: (T['TValue'] | null)[], nullVals?: any[]) => Promise<Vector<T>>) { + describe(`${encode.name} ${name}`, () => { + it(`encodes bools no nulls`, async () => { + const vals = boolsNoNulls(20); + validateVector(vals, await encode(vals, []), []); + }); + it(`encodes bools with nulls`, async () => { + const vals = boolsWithNulls(20); + validateVector(vals, await encode(vals, [null]), [null]); + }); + }); + } +}); + +type PrimitiveTypeOpts<T extends DataType> = [ + new (...args: any[]) => T, + (count: number) => (T['TValue'] | null)[], + (count: number) => (T['TValue'] | null)[], + (count: number) => (T['TValue'] | null)[] +]; + +[ + [Int8, int8sNoNulls, int8sWithNulls, int8sWithMaxInts] as PrimitiveTypeOpts<Int8>, + [Int16, int16sNoNulls, int16sWithNulls, int16sWithMaxInts] as PrimitiveTypeOpts<Int16>, + [Int32, int32sNoNulls, int32sWithNulls, int32sWithMaxInts] as PrimitiveTypeOpts<Int32>, + [Uint8, uint8sNoNulls, uint8sWithNulls, uint8sWithMaxInts] as PrimitiveTypeOpts<Uint8>, + [Uint16, uint16sNoNulls, uint16sWithNulls, uint16sWithMaxInts] as PrimitiveTypeOpts<Uint16>, + [Uint32, uint32sNoNulls, uint32sWithNulls, uint32sWithMaxInts] as PrimitiveTypeOpts<Uint32>, +].forEach(([TypeCtor, noNulls, withNulls, withNaNs]) => { + + describe(`${TypeCtor.name}Builder`, () => { + + const typeFactory = () => new TypeCtor(); + const valueName = TypeCtor.name.toLowerCase(); + + runTestsWithEncoder('encodeAll', encodeAll(typeFactory)); + runTestsWithEncoder('encodeEach: 5', encodeEach(typeFactory, 5)); + runTestsWithEncoder('encodeEach: 25', encodeEach(typeFactory, 25)); + runTestsWithEncoder('encodeEach: undefined', encodeEach(typeFactory)); + testDOMStreams && runTestsWithEncoder('encodeEachDOM: 25', encodeEachDOM(typeFactory, 25)); + testNodeStreams && runTestsWithEncoder('encodeEachNode: 25', encodeEachNode(typeFactory, 25)); + + function runTestsWithEncoder<T extends DataType>(name: string, encode: (vals: (T['TValue'] | null)[], nullVals?: any[]) => Promise<Vector<T>>) { + describe(`${name}`, () => { + it(`encodes ${valueName} no nulls`, async () => { + const vals = noNulls(20); + validateVector(vals, await encode(vals, []), []); + }); + it(`encodes ${valueName} with nulls`, async () => { + const vals = withNulls(20); + validateVector(vals, await encode(vals, [null]), [null]); + }); + it(`encodes ${valueName} with MAX_INT`, async () => { + const vals = withNaNs(20); + validateVector(vals, await encode(vals, [0x7fffffff]), [0x7fffffff]); + }); + }); + } + }); +}); + +[ + [Float16, float16sNoNulls, float16sWithNulls, float16sWithNaNs] as PrimitiveTypeOpts<Float16>, + [Float32, float32sNoNulls, float32sWithNulls, float32sWithNaNs] as PrimitiveTypeOpts<Float32>, + [Float64, float64sNoNulls, float64sWithNulls, float64sWithNaNs] as PrimitiveTypeOpts<Float64>, +].forEach(([TypeCtor, noNulls, withNulls, withNaNs]) => { + + describe(`${TypeCtor.name}Builder`, () => { + + const typeFactory = () => new TypeCtor(); + const valueName = TypeCtor.name.toLowerCase(); + + runTestsWithEncoder('encodeAll', encodeAll(typeFactory)); + runTestsWithEncoder('encodeEach: 5', encodeEach(typeFactory, 5)); + runTestsWithEncoder('encodeEach: 25', encodeEach(typeFactory, 25)); + runTestsWithEncoder('encodeEach: undefined', encodeEach(typeFactory)); + testDOMStreams && runTestsWithEncoder('encodeEachDOM: 25', encodeEachDOM(typeFactory, 25)); + testNodeStreams && runTestsWithEncoder('encodeEachNode: 25', encodeEachNode(typeFactory, 25)); + + function runTestsWithEncoder<T extends DataType>(name: string, encode: (vals: (T['TValue'] | null)[], nullVals?: any[]) => Promise<Vector<T>>) { + describe(`${name}`, () => { + it(`encodes ${valueName} no nulls`, async () => { + const vals = noNulls(20); + validateVector(vals, await encode(vals, []), []); + }); + it(`encodes ${valueName} with nulls`, async () => { + const vals = withNulls(20); + validateVector(vals, await encode(vals, [null]), [null]); + }); + it(`encodes ${valueName} with NaNs`, async () => { + const vals = withNaNs(20); + validateVector(vals, await encode(vals, [NaN]), [NaN]); + }); + }); + } + }); +}); + +describe('Float16Builder', () => { + const encode = encodeAll(() => new Float16()); + it(`encodes the weird values`, async () => { + const vals = [0, 5.960464477539063e-8, NaN, 65504, 2, -0]; + validateVector(vals, await encode(vals, []), []); + }); +}); diff --git a/src/arrow/js/test/unit/builders/uint64-tests.ts b/src/arrow/js/test/unit/builders/uint64-tests.ts new file mode 100644 index 000000000..e08e25b5c --- /dev/null +++ b/src/arrow/js/test/unit/builders/uint64-tests.ts @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { util, Vector, DataType, Uint64 } from 'apache-arrow'; +import { + validateVector, + encodeAll, encodeEach, encodeEachDOM, encodeEachNode, + uint64sNoNulls, uint64sWithNulls, uint64sWithMaxInts, +} from './utils'; + +const testDOMStreams = process.env.TEST_DOM_STREAMS === 'true'; +const testNodeStreams = process.env.TEST_NODE_STREAMS === 'true'; + +const typeFactory = () => new Uint64(); +const valueName = `Uint64`.toLowerCase(); +const encode0 = encodeAll(typeFactory); +const encode1 = encodeEach(typeFactory); +const encode2 = encodeEach(typeFactory, 5); +const encode3 = encodeEach(typeFactory, 25); +const encode4 = encodeEachDOM(typeFactory, 25); +const encode5 = encodeEachNode(typeFactory, 25); + +const nulls0: any[] = [0x7fffffff]; +const nulls1: any[] = [0x7fffffff]; +nulls0[0] = new Uint32Array([0x7fffffff, 0x7fffffff]); +nulls1[0] = util.BN.new(nulls0[0])[Symbol.toPrimitive](); + +type ValuesToVector<T extends DataType> = (values: (T['TValue'] | null)[], nullVals?: any[]) => Promise<Vector<T>>; + +function encodeAndValidate<T extends DataType>(encode: ValuesToVector<T>, providedNulls: any[] = [], expectedNulls = providedNulls) { + return (values: any[]) => { + return async () => { + const vector = await encode(values, providedNulls); + const expected = values.map((x) => { + switch (typeof x) { + case 'number': return new Uint32Array([x, 0]); + case 'bigint': return new Uint32Array(new BigUint64Array([x]).buffer); + } + return x ? x.slice() : x; + }); + return validateVector(expected, vector, expectedNulls); + }; + }; +} + +describe(`Uint64Builder`, () => { + describe(`encode single chunk`, () => { + it(`encodes ${valueName} no nulls`, encodeAndValidate(encode0, [], [])(uint64sNoNulls(20))); + it(`encodes ${valueName} with nulls`, encodeAndValidate(encode0, [null], [null])(uint64sWithNulls(20))); + it(`encodes ${valueName} with MAX_INT`, encodeAndValidate(encode0, nulls0, nulls1)(uint64sWithMaxInts(20))); + }); + describe(`encode chunks length default`, () => { + it(`encodes ${valueName} no nulls`, encodeAndValidate(encode1, [], [])(uint64sNoNulls(20))); + it(`encodes ${valueName} with nulls`, encodeAndValidate(encode1, [null], [null])(uint64sWithNulls(20))); + it(`encodes ${valueName} with MAX_INT`, encodeAndValidate(encode1, nulls0, nulls1)(uint64sWithMaxInts(20))); + }); + describe(`encode chunks length 5`, () => { + it(`encodes ${valueName} no nulls`, encodeAndValidate(encode2, [], [])(uint64sNoNulls(20))); + it(`encodes ${valueName} with nulls`, encodeAndValidate(encode2, [null], [null])(uint64sWithNulls(20))); + it(`encodes ${valueName} with MAX_INT`, encodeAndValidate(encode2, nulls0, nulls1)(uint64sWithMaxInts(20))); + }); + describe(`encode chunks length 25`, () => { + it(`encodes ${valueName} no nulls`, encodeAndValidate(encode3, [], [])(uint64sNoNulls(20))); + it(`encodes ${valueName} with nulls`, encodeAndValidate(encode3, [null], [null])(uint64sWithNulls(20))); + it(`encodes ${valueName} with MAX_INT`, encodeAndValidate(encode3, nulls0, nulls1)(uint64sWithMaxInts(20))); + }); + testDOMStreams && describe(`encode chunks length 25, WhatWG stream`, () => { + it(`encodes ${valueName} no nulls`, encodeAndValidate(encode4, [], [])(uint64sNoNulls(20))); + it(`encodes ${valueName} with nulls`, encodeAndValidate(encode4, [null], [null])(uint64sWithNulls(20))); + it(`encodes ${valueName} with MAX_INT`, encodeAndValidate(encode4, nulls0, nulls1)(uint64sWithMaxInts(20))); + }); + testNodeStreams && describe(`encode chunks length 25, NodeJS stream`, () => { + it(`encodes ${valueName} no nulls`, encodeAndValidate(encode5, [], [])(uint64sNoNulls(20))); + it(`encodes ${valueName} with nulls`, encodeAndValidate(encode5, [null], [null])(uint64sWithNulls(20))); + it(`encodes ${valueName} with MAX_INT`, encodeAndValidate(encode5, nulls0, nulls1)(uint64sWithMaxInts(20))); + }); +}); diff --git a/src/arrow/js/test/unit/builders/utf8-tests.ts b/src/arrow/js/test/unit/builders/utf8-tests.ts new file mode 100644 index 000000000..212879ab4 --- /dev/null +++ b/src/arrow/js/test/unit/builders/utf8-tests.ts @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { validateVector } from './utils'; +import { Vector, Utf8 } from 'apache-arrow'; +import { + encodeAll, + encodeEach, + encodeEachDOM, + encodeEachNode, + stringsNoNulls, + stringsWithNAs, + stringsWithNulls, + stringsWithEmpties +} from './utils'; + +const testDOMStreams = process.env.TEST_DOM_STREAMS === 'true'; +const testNodeStreams = process.env.TEST_NODE_STREAMS === 'true'; + +describe('Utf8Builder', () => { + runTestsWithEncoder('encodeAll', encodeAll(() => new Utf8())); + runTestsWithEncoder('encodeEach: 5', encodeEach(() => new Utf8(), 5)); + runTestsWithEncoder('encodeEach: 25', encodeEach(() => new Utf8(), 25)); + runTestsWithEncoder('encodeEach: undefined', encodeEach(() => new Utf8(), void 0)); + testDOMStreams && runTestsWithEncoder('encodeEachDOM: 25', encodeEachDOM(() => new Utf8(), 25)); + testNodeStreams && runTestsWithEncoder('encodeEachNode: 25', encodeEachNode(() => new Utf8(), 25)); +}); + +function runTestsWithEncoder(name: string, encode: (vals: (string | null)[], nullVals?: any[]) => Promise<Vector<Utf8>>) { + describe(`${encode.name} ${name}`, () => { + it(`encodes strings no nulls`, async () => { + const vals = stringsNoNulls(20); + validateVector(vals, await encode(vals, []), []); + }); + it(`encodes strings with nulls`, async () => { + const vals = stringsWithNulls(20); + validateVector(vals, await encode(vals, [null]), [null]); + }); + it(`encodes strings using n/a as the null value rep`, async () => { + const vals = stringsWithNAs(20); + validateVector(vals, await encode(vals, ['n/a']), ['n/a']); + }); + it(`encodes strings using \\0 as the null value rep`, async () => { + const vals = stringsWithEmpties(20); + validateVector(vals, await encode(vals, ['\0']), ['\0']); + }); + }); +} diff --git a/src/arrow/js/test/unit/builders/utils.ts b/src/arrow/js/test/unit/builders/utils.ts new file mode 100644 index 000000000..9bd16fff3 --- /dev/null +++ b/src/arrow/js/test/unit/builders/utils.ts @@ -0,0 +1,221 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import '../../jest-extensions'; +import { from, fromDOMStream, toArray } from 'ix/asynciterable'; +import { fromNodeStream } from 'ix/asynciterable/fromnodestream'; +import 'ix/Ix.node'; +import { util } from 'apache-arrow'; +import { Builder } from 'apache-arrow'; +import { DataType, Vector, Chunked } from 'apache-arrow'; +import randstr from 'randomatic'; + +const rand = Math.random.bind(Math); +const randnulls = <T, TNull = null>(values: T[], n: TNull = <any> null) => values.map((x) => Math.random() > 0.25 ? x : n) as (T | TNull)[]; + +export const randomBytes = (length: number) => fillRandom(Uint8Array, length); +export const randomString = ((opts) => (length: number) => + randstr('?', length, opts) +)({ chars: `abcdefghijklmnopqrstuvwxyz0123456789_` }); + +export const stringsNoNulls = (length = 20) => Array.from({ length }, (_) => randomString(1 + (Math.random() * 19 | 0))); +export const timestamp32sNoNulls = (length = 20, now = Date.now() / 86400000 | 0) => + Array.from({ length }, (_) => (now + (rand() * 10000 * (rand() > 0.5 ? -1 : 1)) | 0) * 86400000); + +export const timestamp64sNoNulls = (length = 20, now = Date.now()) => Array.from({ length }, (_) => { + const ms = now + (rand() * 31557600000 * (rand() > 0.5 ? -1 : 1) | 0); + return new Int32Array([(ms % 4294967296) | 0, (ms / 4294967296) | 0]); +}); + +export const timestamp32sWithNulls = (length = 20) => randnulls(timestamp32sNoNulls(length), null); +export const timestamp64sWithNulls = (length = 20) => randnulls(timestamp64sNoNulls(length), null); +export const timestamp32sWithMaxInts = (length = 20) => randnulls(timestamp32sNoNulls(length), 0x7fffffff); +export const timestamp64sWithMaxInts = (length = 20) => randnulls(timestamp64sNoNulls(length), new Int32Array([0x7fffffff, 0x7fffffff])); + +export const boolsNoNulls = (length = 20) => Array.from({ length }, () => rand() > 0.5); +export const date32sNoNulls = (length = 20) => timestamp32sNoNulls(length).map((x) => new Date(x)); +export const date64sNoNulls = (length = 20) => timestamp64sNoNulls(length).map((x) => new Date(4294967296 * x[1] + (x[0] >>> 0))); +export const int8sNoNulls = (length = 20) => Array.from(new Int8Array(randomBytes(length * Int8Array.BYTES_PER_ELEMENT).buffer)); +export const int16sNoNulls = (length = 20) => Array.from(new Int16Array(randomBytes(length * Int16Array.BYTES_PER_ELEMENT).buffer)); +export const int32sNoNulls = (length = 20) => Array.from(new Int32Array(randomBytes(length * Int32Array.BYTES_PER_ELEMENT).buffer)); +export const int64sNoNulls = (length = 20) => Array.from({ length }, (_, i) => { + const bn = util.BN.new(new Int32Array(randomBytes(2 * 4).buffer)); + // Evenly distribute the three types of arguments we support in the Int64 + // builder + switch (i % 3) { + // Int32Array (util.BN is-a Int32Array) + case 0: return bn; + // BigInt + case 1: return bn[Symbol.toPrimitive](); + // number + case 2: + default: return bn[0]; + } +}); + +export const uint8sNoNulls = (length = 20) => Array.from(new Uint8Array(randomBytes(length * Uint8Array.BYTES_PER_ELEMENT).buffer)); +export const uint16sNoNulls = (length = 20) => Array.from(new Uint16Array(randomBytes(length * Uint16Array.BYTES_PER_ELEMENT).buffer)); +export const uint32sNoNulls = (length = 20) => Array.from(new Uint32Array(randomBytes(length * Uint32Array.BYTES_PER_ELEMENT).buffer)); +export const uint64sNoNulls = (length = 20) => Array.from({ length }, (_, i) => { + const bn = util.BN.new(new Uint32Array(randomBytes(2 * 4).buffer)); + // Evenly distribute the three types of arguments we support in the Uint64 + // builder + switch (i % 3) { + // UInt32Array (util.BN is-a Uint32Array) + case 0: return bn; + // BigInt + case 1: return bn[Symbol.toPrimitive](); + // number + case 2: + default: return bn[0]; + } +}); +export const float16sNoNulls = (length = 20) => Array.from(new Uint16Array(randomBytes(length * Uint16Array.BYTES_PER_ELEMENT).buffer)).map(util.uint16ToFloat64); +export const float32sNoNulls = (length = 20) => Array.from(new Float32Array(randomBytes(length * Float32Array.BYTES_PER_ELEMENT).buffer)); +export const float64sNoNulls = (length = 20) => Array.from(new Float64Array(randomBytes(length * Float64Array.BYTES_PER_ELEMENT).buffer)); + +export const stringsWithNAs = (length = 20) => randnulls(stringsNoNulls(length), 'n/a'); +export const stringsWithNulls = (length = 20) => randnulls(stringsNoNulls(length), null); +export const stringsWithEmpties = (length = 20) => randnulls(stringsNoNulls(length), '\0'); + +export const boolsWithNulls = (length = 20) => randnulls(boolsNoNulls(length), null); +export const date32sWithNulls = (length = 20) => randnulls(date32sNoNulls(length), null); +export const date64sWithNulls = (length = 20) => randnulls(date64sNoNulls(length), null); +export const int8sWithNulls = (length = 20) => randnulls(int8sNoNulls(length), null); +export const int16sWithNulls = (length = 20) => randnulls(int16sNoNulls(length), null); +export const int32sWithNulls = (length = 20) => randnulls(int32sNoNulls(length), null); +export const int64sWithNulls = (length = 20) => randnulls(int64sNoNulls(length), null); +export const uint8sWithNulls = (length = 20) => randnulls(uint8sNoNulls(length), null); +export const uint16sWithNulls = (length = 20) => randnulls(uint16sNoNulls(length), null); +export const uint32sWithNulls = (length = 20) => randnulls(uint32sNoNulls(length), null); +export const uint64sWithNulls = (length = 20) => randnulls(uint64sNoNulls(length), null); +export const float16sWithNulls = (length = 20) => randnulls(float16sNoNulls(length), null); +export const float32sWithNulls = (length = 20) => randnulls(float32sNoNulls(length), null); +export const float64sWithNulls = (length = 20) => randnulls(float64sNoNulls(length), null); + +export const int8sWithMaxInts = (length = 20) => randnulls(int8sNoNulls(length), 0x7fffffff); +export const int16sWithMaxInts = (length = 20) => randnulls(int16sNoNulls(length), 0x7fffffff); +export const int32sWithMaxInts = (length = 20) => randnulls(int32sNoNulls(length), 0x7fffffff); +export const int64sWithMaxInts = (length = 20) => randnulls(int64sNoNulls(length), new Int32Array([0x7fffffff, 0x7fffffff])); +export const uint8sWithMaxInts = (length = 20) => randnulls(uint8sNoNulls(length), 0x7fffffff); +export const uint16sWithMaxInts = (length = 20) => randnulls(uint16sNoNulls(length), 0x7fffffff); +export const uint32sWithMaxInts = (length = 20) => randnulls(uint32sNoNulls(length), 0x7fffffff); +export const uint64sWithMaxInts = (length = 20) => randnulls(uint64sNoNulls(length), new Uint32Array([0x7fffffff, 0x7fffffff])); +export const float16sWithNaNs = (length = 20) => randnulls(float16sNoNulls(length), NaN); +export const float32sWithNaNs = (length = 20) => randnulls(float32sNoNulls(length), NaN); +export const float64sWithNaNs = (length = 20) => randnulls(float64sNoNulls(length), NaN); + +export const duplicateItems = (n: number, xs: (any | null)[]) => { + const out = new Array<string | null>(n); + for (let i = -1, k = xs.length; ++i < n;) { + out[i] = xs[Math.random() * k | 0]; + } + return out; +}; + +export function encodeAll<T extends DataType>(typeFactory: () => T) { + return async function encodeAll<TNull = any>(values: (T['TValue'] | TNull)[], nullValues?: TNull[]) { + const type = typeFactory(); + const builder = Builder.new({ type, nullValues }); + values.forEach(builder.append.bind(builder)); + return builder.finish().toVector(); + }; +} + +export function encodeEach<T extends DataType>(typeFactory: () => T, chunkLen?: number) { + return async function encodeEach<TNull = any>(vals: (T['TValue'] | TNull)[], nullValues?: TNull[]) { + const type = typeFactory(); + const opts = { type, nullValues, highWaterMark: chunkLen }; + const chunks = [...Builder.throughIterable(opts)(vals)]; + return Chunked.concat(...chunks) as Chunked<T>; + }; +} + +export function encodeEachDOM<T extends DataType>(typeFactory: () => T, chunkLen?: number) { + return async function encodeEachDOM<TNull = any>(vals: (T['TValue'] | TNull)[], nullValues?: TNull[]) { + const type = typeFactory(); + const strategy = { highWaterMark: chunkLen }; + const source = from(vals).toDOMStream(); + const builder = Builder.throughDOM({ type, nullValues, readableStrategy: strategy, writableStrategy: strategy }); + const chunks = await fromDOMStream(source.pipeThrough(builder)).pipe(toArray); + return Chunked.concat(...chunks) as Chunked<T>; + }; +} + +export function encodeEachNode<T extends DataType>(typeFactory: () => T, chunkLen?: number) { + return async function encodeEachNode<TNull = any>(vals: (T['TValue'] | TNull)[], nullValues?: TNull[]) { + const type = typeFactory(); + const vals_ = vals.map((x) => x === null ? undefined : x); + const source = from(vals_).toNodeStream({ objectMode: true }); + const nulls_ = nullValues ? nullValues.map((x) => x === null ? undefined : x) : nullValues; + const builder = Builder.throughNode({ type, nullValues: nulls_, highWaterMark: chunkLen }); + const chunks: any[] = await fromNodeStream(source.pipe(builder), chunkLen).pipe(toArray); + return Chunked.concat(...chunks) as Chunked<T>; + }; +} + +const isInt64Null = (nulls: Map<any, any>, x: any) => { + if (ArrayBuffer.isView(x)) { + const bn = util.BN.new<Int32Array>(x as Int32Array); + return nulls.has((<any> bn)[Symbol.toPrimitive]('default')); + } + return false; +}; + +export function validateVector<T extends DataType>(vals: (T['TValue'] | null)[], vec: Vector, nullVals: any[]) { + let i = 0, x: T['TValue'] | null, y: T['TValue'] | null; + const nulls = nullVals.reduce((m, x) => m.set(x, x), new Map()); + try { + for (x of vec) { + if (nulls.has(y = vals[i])) { + expect(x).toBeNull(); + } else if (isInt64Null(nulls, y)) { + expect(x).toBeNull(); + } else { + expect(x).toArrowCompare(y); + } + i++; + } + } catch (e) { + // Uncomment these two lines to catch and debug the value retrieval that failed + // debugger; + // vec.get(i); + throw new Error([ + `${(vec as any).VectorName}[${i}]: ${e?.stack || e}`, + `nulls: [${nullVals.join(', ')}]`, + `values: [${vals.join(', ')}]`, + ].join('\n')); + } +} + +function fillRandom<T extends TypedArrayConstructor>(ArrayType: T, length: number) { + const BPE = ArrayType.BYTES_PER_ELEMENT; + const array = new ArrayType(length); + const max = (2 ** (8 * BPE)) - 1; + for (let i = -1; ++i < length; array[i] = rand() * max * (rand() > 0.5 ? -1 : 1)) { } + return array as InstanceType<T>; +} + +type TypedArrayConstructor = + (typeof Int8Array) | + (typeof Int16Array) | + (typeof Int32Array) | + (typeof Uint8Array) | + (typeof Uint16Array) | + (typeof Uint32Array) | + (typeof Float32Array) | + (typeof Float64Array); diff --git a/src/arrow/js/test/unit/dataframe-tests.ts b/src/arrow/js/test/unit/dataframe-tests.ts new file mode 100644 index 000000000..9e87e372d --- /dev/null +++ b/src/arrow/js/test/unit/dataframe-tests.ts @@ -0,0 +1,282 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import '../jest-extensions'; +import { + predicate, DataFrame, RecordBatch +} from 'apache-arrow'; +import { test_data } from './table-tests'; +import { jest } from '@jest/globals'; + +const { col, lit, custom, and, or, And, Or } = predicate; + +const F32 = 0, I32 = 1, DICT = 2; + +describe(`DataFrame`, () => { + + for (let datum of test_data) { + describe(datum.name, () => { + + describe(`scan()`, () => { + test(`yields all values`, () => { + const df = new DataFrame(datum.table()); + let expected_idx = 0; + df.scan((idx, batch) => { + const columns = batch.schema.fields.map((_, i) => batch.getChildAt(i)!); + expect(columns.map((c) => c.get(idx))).toEqual(values[expected_idx++]); + }); + }); + test(`calls bind function with every batch`, () => { + const df = new DataFrame(datum.table()); + let bind = jest.fn(); + df.scan(() => { }, bind); + for (let batch of df.chunks) { + expect(bind).toHaveBeenCalledWith(batch); + } + }); + }); + describe(`scanReverse()`, () => { + test(`yields all values`, () => { + const df = new DataFrame(datum.table()); + let expected_idx = values.length; + df.scanReverse((idx, batch) => { + const columns = batch.schema.fields.map((_, i) => batch.getChildAt(i)!); + expect(columns.map((c) => c.get(idx))).toEqual(values[--expected_idx]); + }); + }); + test(`calls bind function with every batch`, () => { + const df = new DataFrame(datum.table()); + let bind = jest.fn(); + df.scanReverse(() => { }, bind); + for (let batch of df.chunks) { + expect(bind).toHaveBeenCalledWith(batch); + } + }); + }); + test(`count() returns the correct length`, () => { + const df = new DataFrame(datum.table()); + const values = datum.values(); + expect(df.count()).toEqual(values.length); + }); + test(`getColumnIndex`, () => { + const df = new DataFrame(datum.table()); + expect(df.getColumnIndex('i32')).toEqual(I32); + expect(df.getColumnIndex('f32')).toEqual(F32); + expect(df.getColumnIndex('dictionary')).toEqual(DICT); + }); + const df = new DataFrame(datum.table()); + const values = datum.values(); + let get_i32: (idx: number) => number, get_f32: (idx: number) => number; + const filter_tests = [ + { + name: `filter on f32 >= 0`, + filtered: df.filter(col('f32').ge(0)), + expected: values.filter((row) => row[F32] >= 0) + }, { + name: `filter on 0 <= f32`, + filtered: df.filter(lit(0).le(col('f32'))), + expected: values.filter((row) => 0 <= row[F32]) + }, { + name: `filter on i32 <= 0`, + filtered: df.filter(col('i32').le(0)), + expected: values.filter((row) => row[I32] <= 0) + }, { + name: `filter on 0 >= i32`, + filtered: df.filter(lit(0).ge(col('i32'))), + expected: values.filter((row) => 0 >= row[I32]) + }, { + name: `filter on f32 < 0`, + filtered: df.filter(col('f32').lt(0)), + expected: values.filter((row) => row[F32] < 0) + }, { + name: `filter on i32 > 1 (empty)`, + filtered: df.filter(col('i32').gt(0)), + expected: values.filter((row) => row[I32] > 0) + }, { + name: `filter on f32 <= -.25 || f3 >= .25`, + filtered: df.filter(col('f32').le(-.25).or(col('f32').ge(.25))), + expected: values.filter((row) => row[F32] <= -.25 || row[F32] >= .25) + }, { + name: `filter on !(f32 <= -.25 || f3 >= .25) (not)`, + filtered: df.filter(col('f32').le(-.25).or(col('f32').ge(.25)).not()), + expected: values.filter((row) => !(row[F32] <= -.25 || row[F32] >= .25)) + }, { + name: `filter method combines predicates (f32 >= 0 && i32 <= 0)`, + filtered: df.filter(col('i32').le(0)).filter(col('f32').ge(0)), + expected: values.filter((row) => row[I32] <= 0 && row[F32] >= 0) + }, { + name: `filter on dictionary == 'a'`, + filtered: df.filter(col('dictionary').eq('a')), + expected: values.filter((row) => row[DICT] === 'a') + }, { + name: `filter on 'a' == dictionary (commutativity)`, + filtered: df.filter(lit('a').eq(col('dictionary'))), + expected: values.filter((row) => row[DICT] === 'a') + }, { + name: `filter on dictionary != 'b'`, + filtered: df.filter(col('dictionary').ne('b')), + expected: values.filter((row) => row[DICT] !== 'b') + }, { + name: `filter on f32 >= i32`, + filtered: df.filter(col('f32').ge(col('i32'))), + expected: values.filter((row) => row[F32] >= row[I32]) + }, { + name: `filter on f32 <= i32`, + filtered: df.filter(col('f32').le(col('i32'))), + expected: values.filter((row) => row[F32] <= row[I32]) + }, { + name: `filter on f32*i32 > 0 (custom predicate)`, + filtered: df.filter(custom( + (idx: number) => (get_f32(idx) * get_i32(idx) > 0), + (batch: RecordBatch) => { + get_f32 = col('f32').bind(batch); + get_i32 = col('i32').bind(batch); + })), + expected: values.filter((row) => (row[F32] as number) * (row[I32] as number) > 0) + }, { + name: `filter out all records`, + filtered: df.filter(lit(1).eq(0)), + expected: [] + } + ]; + for (let this_test of filter_tests) { + const { name, filtered, expected } = this_test; + describe(name, () => { + test(`count() returns the correct length`, () => { + expect(filtered.count()).toEqual(expected.length); + }); + describe(`scan()`, () => { + test(`iterates over expected values`, () => { + let expected_idx = 0; + filtered.scan((idx, batch) => { + const columns = batch.schema.fields.map((_, i) => batch.getChildAt(i)!); + expect(columns.map((c) => c.get(idx))).toEqual(expected[expected_idx++]); + }); + }); + test(`calls bind function lazily`, () => { + let bind = jest.fn(); + filtered.scan(() => { }, bind); + if (expected.length) { + expect(bind).toHaveBeenCalled(); + } else { + expect(bind).not.toHaveBeenCalled(); + } + }); + }); + describe(`scanReverse()`, () => { + test(`iterates over expected values in reverse`, () => { + let expected_idx = expected.length; + filtered.scanReverse((idx, batch) => { + const columns = batch.schema.fields.map((_, i) => batch.getChildAt(i)!); + expect(columns.map((c) => c.get(idx))).toEqual(expected[--expected_idx]); + }); + }); + test(`calls bind function lazily`, () => { + let bind = jest.fn(); + filtered.scanReverse(() => { }, bind); + if (expected.length) { + expect(bind).toHaveBeenCalled(); + } else { + expect(bind).not.toHaveBeenCalled(); + } + }); + }); + }); + } + test(`countBy on dictionary returns the correct counts`, () => { + // Make sure countBy works both with and without the Col wrapper + // class + let expected: { [key: string]: number } = { 'a': 0, 'b': 0, 'c': 0 }; + for (let row of values) { + expected[row[DICT]] += 1; + } + + expect(df.countBy(col('dictionary')).toJSON()).toEqual(expected); + expect(df.countBy('dictionary').toJSON()).toEqual(expected); + }); + test(`countBy on dictionary with filter returns the correct counts`, () => { + let expected: { [key: string]: number } = { 'a': 0, 'b': 0, 'c': 0 }; + for (let row of values) { + if (row[I32] === 1) { expected[row[DICT]] += 1; } + } + + expect(df.filter(col('i32').eq(1)).countBy('dictionary').toJSON()).toEqual(expected); + }); + test(`countBy on non dictionary column throws error`, () => { + expect(() => { df.countBy('i32'); }).toThrow(); + expect(() => { df.filter(col('dict').eq('a')).countBy('i32'); }).toThrow(); + }); + test(`countBy on non-existent column throws error`, () => { + expect(() => { df.countBy('FAKE' as any); }).toThrow(); + }); + test(`table.select() basic tests`, () => { + let selected = df.select('f32', 'dictionary'); + expect(selected.schema.fields).toHaveLength(2); + expect(selected.schema.fields[0]).toEqual(df.schema.fields[0]); + expect(selected.schema.fields[1]).toEqual(df.schema.fields[2]); + + expect(selected).toHaveLength(values.length); + let idx = 0, expected_row; + for (let row of selected) { + expected_row = values[idx++]; + expect(row.f32).toEqual(expected_row[F32]); + expect(row.dictionary).toEqual(expected_row[DICT]); + } + }); + test(`table.filter(..).count() on always false predicates returns 0`, () => { + expect(df.filter(col('i32').ge(100)).count()).toEqual(0); + expect(df.filter(col('dictionary').eq('z')).count()).toEqual(0); + }); + describe(`lit-lit comparison`, () => { + test(`always-false count() returns 0`, () => { + expect(df.filter(lit('abc').eq('def')).count()).toEqual(0); + expect(df.filter(lit(0).ge(1)).count()).toEqual(0); + }); + test(`always-true count() returns length`, () => { + expect(df.filter(lit('abc').eq('abc')).count()).toEqual(df.length); + expect(df.filter(lit(-100).le(0)).count()).toEqual(df.length); + }); + }); + describe(`col-col comparison`, () => { + test(`always-false count() returns 0`, () => { + expect(df.filter(col('dictionary').eq(col('i32'))).count()).toEqual(0); + }); + test(`always-true count() returns length`, () => { + expect(df.filter(col('dictionary').eq(col('dictionary'))).count()).toEqual(df.length); + }); + }); + }); + } +}); + +describe(`Predicate`, () => { + const p1 = col('a').gt(100); + const p2 = col('a').lt(1000); + const p3 = col('b').eq('foo'); + const p4 = col('c').eq('bar'); + const expected = [p1, p2, p3, p4]; + test(`and flattens children`, () => { + expect(and(p1, p2, p3, p4).children).toEqual(expected); + expect(and(p1.and(p2), new And(p3, p4)).children).toEqual(expected); + expect(and(p1.and(p2, p3, p4)).children).toEqual(expected); + }); + test(`or flattens children`, () => { + expect(or(p1, p2, p3, p4).children).toEqual(expected); + expect(or(p1.or(p2), new Or(p3, p4)).children).toEqual(expected); + expect(or(p1.or(p2, p3, p4)).children).toEqual(expected); + }); +}); diff --git a/src/arrow/js/test/unit/generated-data-tests.ts b/src/arrow/js/test/unit/generated-data-tests.ts new file mode 100644 index 000000000..ab1276f76 --- /dev/null +++ b/src/arrow/js/test/unit/generated-data-tests.ts @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import '../jest-extensions'; +import * as generate from '../generate-test-data'; +import { validateTable, validateRecordBatch, validateVector } from './generated-data-validators'; + +describe('Generated Test Data', () => { + describe('Table', () => { validateTable(generate.table([100, 150, 75])).run(); }); + describe('RecordBatch', () => { validateRecordBatch(generate.recordBatch()).run(); }); + describe('NullVector', () => { validateVector(generate.null_()).run(); }); + describe('BoolVector', () => { validateVector(generate.bool()).run(); }); + describe('Int8Vector', () => { validateVector(generate.int8()).run(); }); + describe('Int16Vector', () => { validateVector(generate.int16()).run(); }); + describe('Int32Vector', () => { validateVector(generate.int32()).run(); }); + describe('Int64Vector', () => { validateVector(generate.int64()).run(); }); + describe('Uint8Vector', () => { validateVector(generate.uint8()).run(); }); + describe('Uint16Vector', () => { validateVector(generate.uint16()).run(); }); + describe('Uint32Vector', () => { validateVector(generate.uint32()).run(); }); + describe('Uint64Vector', () => { validateVector(generate.uint64()).run(); }); + describe('Float16Vector', () => { validateVector(generate.float16()).run(); }); + describe('Float32Vector', () => { validateVector(generate.float32()).run(); }); + describe('Float64Vector', () => { validateVector(generate.float64()).run(); }); + describe('Utf8Vector', () => { validateVector(generate.utf8()).run(); }); + describe('BinaryVector', () => { validateVector(generate.binary()).run(); }); + describe('FixedSizeBinaryVector', () => { validateVector(generate.fixedSizeBinary()).run(); }); + describe('DateDayVector', () => { validateVector(generate.dateDay()).run(); }); + describe('DateMillisecondVector', () => { validateVector(generate.dateMillisecond()).run(); }); + describe('TimestampSecondVector', () => { validateVector(generate.timestampSecond()).run(); }); + describe('TimestampMillisecondVector', () => { validateVector(generate.timestampMillisecond()).run(); }); + describe('TimestampMicrosecondVector', () => { validateVector(generate.timestampMicrosecond()).run(); }); + describe('TimestampNanosecondVector', () => { validateVector(generate.timestampNanosecond()).run(); }); + describe('TimeSecondVector', () => { validateVector(generate.timeSecond()).run(); }); + describe('TimeMillisecondVector', () => { validateVector(generate.timeMillisecond()).run(); }); + describe('TimeMicrosecondVector', () => { validateVector(generate.timeMicrosecond()).run(); }); + describe('TimeNanosecondVector', () => { validateVector(generate.timeNanosecond()).run(); }); + describe('DecimalVector', () => { validateVector(generate.decimal()).run(); }); + describe('ListVector', () => { validateVector(generate.list()).run(); }); + describe('StructVector', () => { validateVector(generate.struct()).run(); }); + describe('DenseUnionVector', () => { validateVector(generate.denseUnion()).run(); }); + describe('SparseUnionVector', () => { validateVector(generate.sparseUnion()).run(); }); + describe('DictionaryVector', () => { validateVector(generate.dictionary()).run(); }); + describe('IntervalDayTimeVector', () => { validateVector(generate.intervalDayTime()).run(); }); + describe('IntervalYearMonthVector', () => { validateVector(generate.intervalYearMonth()).run(); }); + describe('FixedSizeListVector', () => { validateVector(generate.fixedSizeList()).run(); }); + describe('MapVector', () => { validateVector(generate.map()).run(); }); +}); diff --git a/src/arrow/js/test/unit/generated-data-validators.ts b/src/arrow/js/test/unit/generated-data-validators.ts new file mode 100644 index 000000000..910386d4a --- /dev/null +++ b/src/arrow/js/test/unit/generated-data-validators.ts @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import '../jest-extensions'; +import { + GeneratedTable, + GeneratedRecordBatch, + GeneratedVector +} from '../generate-test-data'; + +import { util } from 'apache-arrow'; +const { createElementComparator: compare } = util; + +type DeferredTest = { description: string; tests?: DeferredTest[]; run: (...args: any[]) => any }; + +function deferTest(description: string, run: (...args: any[]) => any) { + return { description, run: () => test(description, run) } as DeferredTest; +} + +function deferDescribe(description: string, tests: DeferredTest | DeferredTest[]) { + const t = (Array.isArray(tests) ? tests : [tests]).filter(Boolean); + return { description, tests: t, run: () => describe(description, () => { t.forEach((x) => x.run()); } ) }; +} + +export function validateTable({ keys, rows, cols, rowBatches, colBatches, keyBatches, table }: GeneratedTable) { + return deferDescribe(`Table: ${table.schema}`, ([] as DeferredTest[]).concat( + validateVector({ values: rows, vector: table }), + table.chunks.map((recordBatch, i) => + deferDescribe(`recordBatch ${i}`, validateRecordBatch({ + keys: keyBatches[i], rows: rowBatches[i], cols: colBatches[i], recordBatch + })) + ), + table.schema.fields.map((field, i) => + deferDescribe(`column ${i}: ${field}`, validateVector({ + keys: keys()[i], + values: () => cols()[i], + vector: table.getColumnAt(i)! + })) + ) + )); +} + +export function validateRecordBatch({ rows, cols, keys, recordBatch }: GeneratedRecordBatch) { + return deferDescribe(`RecordBatch: ${recordBatch.schema}`, ([] as DeferredTest[]).concat( + validateVector({ values: rows, vector: recordBatch }), + recordBatch.schema.fields.map((field, i) => + deferDescribe(`Field: ${field}`, validateVector({ + keys: keys()[i], + values: () => cols()[i], + vector: recordBatch.getChildAt(i)! + })) + ) + )); +} + +export function validateVector({ values: createTestValues, vector, keys }: GeneratedVector, sliced = false) { + + const values = createTestValues(); + const suites = [ + deferDescribe(`Validate ${vector.type} (sliced=${sliced})`, [ + deferTest(`length is correct`, () => { + expect(vector).toHaveLength(values.length); + }), + deferTest(`gets expected values`, () => { + expect.hasAssertions(); + let i = -1, n = vector.length, actual, expected; + try { + while (++i < n) { + actual = vector.get(i); + expected = values[i]; + expect(actual).toArrowCompare(expected); + } + } catch (e) { throw new Error(`${vector}[${i}]: ${e}`); } + }), + (keys && keys.length > 0) && deferTest(`dictionary indices should match`, () => { + expect.hasAssertions(); + let indices = (vector as any).indices; + let i = -1, n = indices.length; + try { + while (++i < n) { + indices.isValid(i) + ? expect(indices.get(i)).toBe(keys[i]) + : expect(indices.get(i)).toBeNull(); + } + } catch (e) { throw new Error(`${indices}[${i}]: ${e}`); } + }) || null as any as DeferredTest, + deferTest(`sets expected values`, () => { + expect.hasAssertions(); + let i = -1, n = vector.length, actual, expected; + try { + while (++i < n) { + expected = vector.get(i); + vector.set(i, expected); + actual = vector.get(i); + expect(actual).toArrowCompare(expected); + } + } catch (e) { throw new Error(`${vector}[${i}]: ${e}`); } + }), + deferTest(`iterates expected values`, () => { + expect.hasAssertions(); + let i = -1, actual, expected; + try { + for (actual of vector) { + expected = values[++i]; + expect(actual).toArrowCompare(expected); + } + } catch (e) { throw new Error(`${vector}[${i}]: ${e}`); } + }), + deferTest(`indexOf returns expected values`, () => { + expect.hasAssertions(); + let i = -1, n = vector.length; + const shuffled = shuffle(values); + let value: any, actual, expected; + try { + while (++i < n) { + value = shuffled[i]; + actual = vector.indexOf(value); + expected = values.findIndex(compare(value)); + expect(actual).toBe(expected); + } + // I would be pretty surprised if randomatic ever generates these values + expect(vector.indexOf('purple elephants')).toBe(-1); + expect(vector.indexOf('whistling wombats')).toBe(-1); + expect(vector.indexOf('carnivorous novices')).toBe(-1); + } catch (e) { throw new Error(`${vector}[${i}]: ${e}`); } + }) + ]) + ] as DeferredTest[]; + + if (!sliced) { + const begin = (values.length * .25) | 0; + const end = (values.length * .75) | 0; + suites.push( + // test slice with no args + validateVector({ + vector: vector.slice(), + values: () => values.slice(), + keys: keys ? keys.slice() : undefined + }, true), + // test slicing half the array + validateVector({ + vector: vector.slice(begin, end), + values: () => values.slice(begin, end), + keys: keys ? keys.slice(begin, end) : undefined + }, true), + // test concat each end together + validateVector({ + vector: vector.slice(0, begin).concat(vector.slice(end)), + values: () => values.slice(0, begin).concat(values.slice(end)), + keys: keys ? [...keys.slice(0, begin), ...keys.slice(end)] : undefined + }, true) + ); + + return deferDescribe(`Vector`, suites); + } + + return suites[0]; +} + +function shuffle(input: any[]) { + const result = input.slice(); + let j, tmp, i = result.length; + while (--i > 0) { + j = (Math.random() * (i + 1)) | 0; + tmp = result[i]; + result[i] = result[j]; + result[j] = tmp; + } + return result; +} diff --git a/src/arrow/js/test/unit/int-tests.ts b/src/arrow/js/test/unit/int-tests.ts new file mode 100644 index 000000000..15c75e1a1 --- /dev/null +++ b/src/arrow/js/test/unit/int-tests.ts @@ -0,0 +1,241 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import * as Arrow from 'apache-arrow'; +const { Int64, Uint64, Int128 } = Arrow.util; + +describe(`Uint64`, () => { + test(`gets expected high/low bytes`, () => { + let i = new Uint64(new Uint32Array([5, 0])); + expect(i.high()).toEqual(0); + expect(i.low()).toEqual(5); + }); + test(`adds 32-bit numbers`, () => { + let a = new Uint64(new Uint32Array([5, 0])); + let b = new Uint64(new Uint32Array([9, 0])); + let expected = new Uint64(new Uint32Array([14, 0])); + expect(a.plus(b)).toEqual(expected); + }); + test(`addition overflows 32-bit numbers`, () => { + let a = new Uint64(new Uint32Array([0xffffffff, 0])); + let b = new Uint64(new Uint32Array([9, 0])); + let expected = new Uint64(new Uint32Array([8, 1])); + expect(a.plus(b)).toEqual(expected); + }); + test(`multiplies 32-bit numbers`, () => { + let a = new Uint64(new Uint32Array([5, 0])); + let b = new Uint64(new Uint32Array([9, 0])); + let expected = new Uint64(new Uint32Array([45, 0])); + expect(a.times(b)).toEqual(expected); + }); + test(`multiplication overflows 32-bit numbers`, () => { + let a = new Uint64(new Uint32Array([0x80000000, 0])); + let b = new Uint64(new Uint32Array([3, 0])); + let expected = new Uint64(new Uint32Array([0x80000000, 1])); + expect(a.times(b)).toEqual(expected); + }); + test(`multiplication is associative`, () => { + let a = new Uint64(new Uint32Array([0x80000000, 0])); + let b = new Uint64(new Uint32Array([3, 0])); + expect(Uint64.multiply(a, b)).toEqual(Uint64.multiply(b,a)); + }); + test(`lessThan works on 32-bit numbers`, () => { + let a = new Uint64(new Uint32Array([0x0000abcd, 0])); + let b = new Uint64(new Uint32Array([0x0000abcf, 0])); + expect(a.lessThan(b)).toBeTruthy(); + }); + test(`lessThan works on 64-bit numbers`, () => { + let a = new Uint64(new Uint32Array([123, 32])); + let b = new Uint64(new Uint32Array([568, 32])); + expect(a.lessThan(b)).toBeTruthy(); + }); + test(`fromString parses string`, () => { + expect(Uint64.fromString('6789123456789')).toEqual(new Int64(new Uint32Array([0xb74abf15, 0x62c]))); + }); + test(`fromString parses big (full unsigned 64-bit) string`, () => { + expect(Uint64.fromString('18364758544493064720')).toEqual(new Uint64(new Uint32Array([0x76543210, 0xfedcba98]))); + }); + test(`fromNumber converts 53-ish bit number`, () => { + expect(Uint64.fromNumber(8086463330923024)).toEqual(new Uint64(new Uint32Array([0x76543210, 0x001cba98]))); + }); +}); + +describe(`Int64`, () => { + test(`gets expected high/low bytes`, () => { + let i = new Int64(new Uint32Array([5, 0])); + expect(i.high()).toEqual(0); + expect(i.low()).toEqual(5); + }); + test(`adds 32-bit numbers`, () => { + let a = new Int64(new Uint32Array([5, 0])); + let b = new Int64(new Uint32Array([9, 0])); + let expected = new Int64(new Uint32Array([14, 0])); + expect(a.plus(b)).toEqual(expected); + }); + test(`adds negative 32-bit numbers`, () => { + let a = new Int64(new Uint32Array([56789 , 0])); + let b = new Int64(new Uint32Array([-66789, -1])); + let expected = new Int64(new Uint32Array([-10000, -1])); + expect(a.plus(b)).toEqual(expected); + }); + test(`addition overflows 32-bit numbers`, () => { + let a = new Int64(new Uint32Array([0xffffffff, 0])); + let b = new Int64(new Uint32Array([9, 0])); + let expected = new Int64(new Uint32Array([8, 1])); + expect(a.plus(b)).toEqual(expected); + }); + test(`multiplies 32-bit numbers`, () => { + let a = new Int64(new Uint32Array([5, 0])); + let b = new Int64(new Uint32Array([9, 0])); + let expected = new Int64(new Uint32Array([45, 0])); + expect(a.times(b)).toEqual(expected); + }); + test(`multiplication overflows 32-bit numbers`, () => { + let a = new Int64(new Uint32Array([0x80000000, 0])); + let b = new Int64(new Uint32Array([3, 0])); + let expected = new Int64(new Uint32Array([0x80000000, 1])); + expect(a.times(b)).toEqual(expected); + }); + test(`multiplication works on negative numbers`, () => { + let a = new Int64(new Uint32Array([-5, -1])); + let b = new Int64(new Uint32Array([-100, -1])); + expect(a.times(b)).toEqual(new Int64(new Uint32Array([ 500, 0]))); + expect(a.times(b)).toEqual(new Int64(new Uint32Array([ -50000, -1]))); + expect(a.times(b)).toEqual(new Int64(new Uint32Array([5000000, 0]))); + }); + test(`multiplication is associative`, () => { + let a = new Int64(new Uint32Array([0x80000000, 0])); + let b = new Int64(new Uint32Array([3, 0])); + expect(Int64.multiply(a, b)).toEqual(Int64.multiply(b,a)); + }); + test(`lessThan works on 32-bit numbers`, () => { + let a = new Int64(new Uint32Array([0x0000abcd, 0])); + let b = new Int64(new Uint32Array([0x0000abcf, 0])); + expect(a.lessThan(b)).toBeTruthy(); + }); + test(`lessThan works on 64-bit numbers`, () => { + let a = new Int64(new Uint32Array([123, 32])); + let b = new Int64(new Uint32Array([568, 32])); + expect(a.lessThan(b)).toBeTruthy(); + }); + test(`lessThan works on negative numbers`, () => { + let a = new Int64(new Uint32Array([0, -158])); + let b = new Int64(new Uint32Array([-3, -1])); + expect(a.lessThan(b)).toBeTruthy(); + }); + test(`lessThan works on mixed numbers`, () => { + let a = new Int64(new Uint32Array([-3, -1])); + let b = new Int64(new Uint32Array([ 0, 3])); + expect(a.lessThan(b)).toBeTruthy(); + }); + test(`negate works on 32-bit number`, () => { + expect (new Int64(new Uint32Array([123456, 0])).negate()).toEqual(new Int64(new Uint32Array([-123456, -1]))); + }); + test(`double negation is noop`, () => { + let test = new Int64(new Uint32Array([6789, 12345])); + let expected = new Int64(new Uint32Array([6789, 12345])); + expect(test.negate().negate()).toEqual(expected); + }); + test(`negate works on 64-bit number`, () => { + expect (new Int64(new Uint32Array([0xb74abf15, 0x62c])).negate()).toEqual(new Int64(new Uint32Array([0x48b540eb, 0xfffff9d3]))); + }); + test(`fromString parses string`, () => { + expect(Int64.fromString('6789123456789')).toEqual(new Int64(new Uint32Array([0xb74abf15, 0x62c]))); + }); + test(`fromString parses negative string`, () => { + expect(Int64.fromString('-6789123456789')).toEqual(new Int64(new Uint32Array([0x48b540eb, 0xfffff9d3]))); + }); + test(`fromNumber converts 53-ish bit number`, () => { + expect(Int64.fromNumber(8086463330923024)).toEqual(new Int64(new Uint32Array([0x76543210, 0x001cba98]))); + expect(Int64.fromNumber(-8086463330923024)).toEqual(new Int64(new Uint32Array([0x89abcdf0, 0xffe34567]))); + }); +}); + +describe(`Int128`, () => { + test(`gets expected bytes`, () => { + let i = new Int128(new Uint32Array([4, 3, 2, 1])); + expect(i.high().high()).toEqual(1); + expect(i.high().low() ).toEqual(2); + expect(i.low().high() ).toEqual(3); + expect(i.low().low() ).toEqual(4); + }); + test(`adds 32-bit numbers`, () => { + let a = new Int128(new Uint32Array([5, 0, 0, 0])); + let b = new Int128(new Uint32Array([9, 0, 0, 0])); + let expected = new Int128(new Uint32Array([14, 0, 0, 0])); + expect(a.plus(b)).toEqual(expected); + }); + test(`adds negative 32-bit numbers`, () => { + let a = new Int128(new Uint32Array([56789 , 0, 0, 0])); + let b = new Int128(new Uint32Array([-66789, -1, -1, -1])); + let expected = new Int128(new Uint32Array([-10000, -1, -1, -1])); + expect(a.plus(b)).toEqual(expected); + }); + test(`addition overflows 32-bit numbers`, () => { + let a = new Int128(new Uint32Array([0xffffffff, 0, 0, 0])); + let b = new Int128(new Uint32Array([9, 0, 0, 0])); + let expected = new Int128(new Uint32Array([8, 1, 0, 0])); + expect(a.plus(b)).toEqual(expected); + }); + test(`multiplies 32-bit numbers`, () => { + let a = new Int128(new Uint32Array([5, 0, 0, 0])); + let b = new Int128(new Uint32Array([9, 0, 0, 0])); + let expected = new Int128(new Uint32Array([45, 0, 0, 0])); + expect(a.times(b)).toEqual(expected); + }); + test(`multiplication overflows 32-bit numbers`, () => { + let a = new Int128(new Uint32Array([0x80000000, 0, 0, 0])); + let b = new Int128(new Uint32Array([3, 0, 0, 0])); + let expected = new Int128(new Uint32Array([0x80000000, 1, 0, 0])); + expect(a.times(b)).toEqual(expected); + }); + test(`multiplication works on negative numbers`, () => { + let a = new Int128(new Uint32Array([-5, -1, -1, -1])); + let b = new Int128(new Uint32Array([-100, -1, -1, -1])); + expect(a.times(b)).toEqual(new Int128(new Uint32Array([ 500, 0, 0, 0]))); + expect(a.times(b)).toEqual(new Int128(new Uint32Array([ -50000, -1, -1, -1]))); + expect(a.times(b)).toEqual(new Int128(new Uint32Array([5000000, 0, 0, 0]))); + }); + test(`multiplication is associative`, () => { + let a = new Int128(new Uint32Array([4, 3, 2, 1])); + let b = new Int128(new Uint32Array([3, 0, 0, 0])); + expect(Int128.multiply(a, b)).toEqual(Int128.multiply(b,a)); + }); + test(`multiplication can produce 128-bit number`, () => { + let a = new Int128(new Uint32Array([0, 0xf0000000, 0, 0])); + let b = new Int128(new Uint32Array([0, 0x10000000, 0, 0])); + expect(a.times(b)).toEqual(new Int128(new Uint32Array([0x00000000, 0x00000000, 0x00000000, 0xf000000]))); + }); + test(`fromString parses string`, () => { + expect(Int128.fromString('1002111867823618826746863804903129070')) + .toEqual(new Int64(new Uint32Array([0x00c0ffee, + 0x00c0ffee, + 0x00c0ffee, + 0x00c0ffee]))); + }); + test(`fromString parses negative string`, () => { + expect(Int128.fromString('-12345678901234567890123456789012345678')) + .toEqual(new Int64(new Uint32Array([0x21c70cb2, + 0x3bb66faf, + 0x0ffdccec, + 0xf6b64f09]))); + }); + test(`fromNumber converts 53-ish bit number`, () => { + expect(Int128.fromNumber(8086463330923024)).toEqual(new Int128(new Uint32Array([0x76543210, 0x001cba98, 0, 0]))); + expect(Int128.fromNumber(-8086463330923024)).toEqual(new Int128(new Uint32Array([0x89abcdf0, 0xffe34567, 0xffffffff, 0xffffffff]))); + }); +}); diff --git a/src/arrow/js/test/unit/ipc/helpers.ts b/src/arrow/js/test/unit/ipc/helpers.ts new file mode 100644 index 000000000..9fccefec9 --- /dev/null +++ b/src/arrow/js/test/unit/ipc/helpers.ts @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import '../../jest-extensions'; + +import { + Table, + RecordBatchWriter, + RecordBatchFileWriter, + RecordBatchJSONWriter, + RecordBatchStreamWriter, +} from 'apache-arrow'; + +import * as fs from 'fs'; +import { fs as memfs } from 'memfs'; +import { Readable, PassThrough } from 'stream'; +import randomatic from 'randomatic'; + +export abstract class ArrowIOTestHelper { + + constructor(public table: Table) {} + + public static file(table: Table) { return new ArrowFileIOTestHelper(table); } + public static json(table: Table) { return new ArrowJsonIOTestHelper(table); } + public static stream(table: Table) { return new ArrowStreamIOTestHelper(table); } + + protected abstract writer(table: Table): RecordBatchWriter; + protected async filepath(table: Table): Promise<fs.PathLike> { + const path = `/${randomatic('a0', 20)}.arrow`; + const data = await this.writer(table).toUint8Array(); + await memfs.promises.writeFile(path, data); + return path; + } + + buffer(testFn: (buffer: Uint8Array) => void | Promise<void>) { + return async () => { + expect.hasAssertions(); + await testFn(await this.writer(this.table).toUint8Array()); + }; + } + iterable(testFn: (iterable: Generator<Uint8Array>) => void | Promise<void>) { + return async () => { + expect.hasAssertions(); + await testFn(chunkedIterable(await this.writer(this.table).toUint8Array())); + }; + } + asyncIterable(testFn: (asyncIterable: AsyncGenerator<Uint8Array>) => void | Promise<void>) { + return async () => { + expect.hasAssertions(); + await testFn(asyncChunkedIterable(await this.writer(this.table).toUint8Array())); + }; + } + fsFileHandle(testFn: (handle: fs.promises.FileHandle) => void | Promise<void>) { + return async () => { + expect.hasAssertions(); + const path = await this.filepath(this.table); + await testFn(<any> await memfs.promises.open(path, 'r')); + await memfs.promises.unlink(path); + }; + } + fsReadableStream(testFn: (stream: fs.ReadStream) => void | Promise<void>) { + return async () => { + expect.hasAssertions(); + const path = await this.filepath(this.table); + await testFn(<any> memfs.createReadStream(path)); + await memfs.promises.unlink(path); + }; + } + nodeReadableStream(testFn: (stream: NodeJS.ReadableStream) => void | Promise<void>) { + return async () => { + expect.hasAssertions(); + const sink = new PassThrough(); + sink.end(await this.writer(this.table).toUint8Array()); + await testFn(sink); + }; + } + whatwgReadableStream(testFn: (stream: ReadableStream) => void | Promise<void>) { + return async () => { + expect.hasAssertions(); + const path = await this.filepath(this.table); + await testFn(nodeToDOMStream(memfs.createReadStream(path))); + await memfs.promises.unlink(path); + }; + } + whatwgReadableByteStream(testFn: (stream: ReadableStream) => void | Promise<void>) { + return async () => { + expect.hasAssertions(); + const path = await this.filepath(this.table); + await testFn(nodeToDOMStream(memfs.createReadStream(path), { type: 'bytes' })); + await memfs.promises.unlink(path); + }; + } +} + +class ArrowFileIOTestHelper extends ArrowIOTestHelper { + constructor(table: Table) { super(table); } + protected writer(table: Table) { + return RecordBatchFileWriter.writeAll(table); + } +} + +class ArrowJsonIOTestHelper extends ArrowIOTestHelper { + constructor(table: Table) { super(table); } + protected writer(table: Table) { + return RecordBatchJSONWriter.writeAll(table); + } +} + +class ArrowStreamIOTestHelper extends ArrowIOTestHelper { + constructor(table: Table) { super(table); } + protected writer(table: Table) { + return RecordBatchStreamWriter.writeAll(table); + } +} + +export function* chunkedIterable(buffer: Uint8Array) { + let offset = 0, size = 0; + while (offset < buffer.byteLength) { + size = yield buffer.subarray(offset, offset += + (isNaN(+size) ? buffer.byteLength - offset : size)); + } +} + +export async function* asyncChunkedIterable(buffer: Uint8Array) { + let offset = 0, size = 0; + while (offset < buffer.byteLength) { + size = yield buffer.subarray(offset, offset += + (isNaN(+size) ? buffer.byteLength - offset : size)); + } +} + +export async function concatBuffersAsync(iterator: AsyncIterable<Uint8Array> | ReadableStream) { + if (iterator instanceof ReadableStream) { + iterator = readableDOMStreamToAsyncIterator(iterator); + } + let chunks = [], total = 0; + for await (const chunk of iterator) { + chunks.push(chunk); + total += chunk.byteLength; + } + return chunks.reduce((x, buffer) => { + x.buffer.set(buffer, x.offset); + x.offset += buffer.byteLength; + return x; + }, { offset: 0, buffer: new Uint8Array(total) }).buffer; +} + +export async function* readableDOMStreamToAsyncIterator<T>(stream: ReadableStream<T>) { + // Get a lock on the stream + const reader = stream.getReader(); + try { + while (true) { + // Read from the stream + const { done, value } = await reader.read(); + // Exit if we're done + if (done) { break; } + // Else yield the chunk + yield value as T; + } + } finally { + try { stream.locked && reader.releaseLock(); } catch (e) {} + } +} + +export function nodeToDOMStream<T = any>(stream: NodeJS.ReadableStream, opts: any = {}) { + stream = new Readable((stream as any)._readableState).wrap(stream); + return new ReadableStream<T>({ + ...opts, + start(controller) { + stream.pause(); + stream.on('data', (chunk) => { + controller.enqueue(chunk); + stream.pause(); + }); + stream.on('end', () => controller.close()); + stream.on('error', e => controller.error(e)); + }, + pull() { stream.resume(); }, + cancel(reason) { + stream.pause(); + if (typeof (stream as any).cancel === 'function') { + return (stream as any).cancel(reason); + } else if (typeof (stream as any).destroy === 'function') { + return (stream as any).destroy(reason); + } + } + }); +} diff --git a/src/arrow/js/test/unit/ipc/message-reader-tests.ts b/src/arrow/js/test/unit/ipc/message-reader-tests.ts new file mode 100644 index 000000000..c48aa2ce1 --- /dev/null +++ b/src/arrow/js/test/unit/ipc/message-reader-tests.ts @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// import * as fs from 'fs'; +import { + generateRandomTables, + // generateDictionaryTables +} from '../../data/tables'; + +import { ArrowIOTestHelper } from './helpers'; +import { MessageReader, AsyncMessageReader } from 'apache-arrow'; + +for (const table of generateRandomTables([10, 20, 30])) { + + const io = ArrowIOTestHelper.stream(table); + const name = `[\n ${table.schema.fields.join(',\n ')}\n]`; + const numMessages = table.chunks.reduce((numMessages, batch) => { + return numMessages + + /* recordBatch message */ 1 + + /* dictionary messages */ batch.dictionaries.size; + }, /* schema message */ 1); + + const validate = validateMessageReader.bind(0, numMessages); + const validateAsync = validateAsyncMessageReader.bind(0, numMessages); + + describe(`MessageReader (${name})`, () => { + describe(`should read all Messages`, () => { + test(`Uint8Array`, io.buffer(validate)); + test(`Iterable`, io.iterable(validate)); + }); + }); + + describe(`AsyncMessageReader (${name})`, () => { + describe(`should read all Messages`, () => { + test('AsyncIterable', io.asyncIterable(validateAsync)); + test('fs.FileHandle', io.fsFileHandle(validateAsync)); + test('fs.ReadStream', io.fsReadableStream(validateAsync)); + test('stream.Readable', io.nodeReadableStream(validateAsync)); + test('whatwg.ReadableStream', io.whatwgReadableStream(validateAsync)); + test('whatwg.ReadableByteStream', io.whatwgReadableByteStream(validateAsync)); + }); + }); +} + +export function validateMessageReader(numMessages: number, source: any) { + const reader = new MessageReader(source); + let index = 0; + for (let message of reader) { + + if (index === 0) { + expect(message.isSchema()).toBe(true); + expect(message.bodyLength).toBe(0); + } else { + expect(message.isSchema()).toBe(false); + expect(message.isRecordBatch() || message.isDictionaryBatch()).toBe(true); + } + + try { + expect(message.bodyLength % 8).toBe(0); + } catch (e) { throw new Error(`bodyLength: ${e}`); } + + const body = reader.readMessageBody(message.bodyLength); + expect(body).toBeInstanceOf(Uint8Array); + expect(body.byteLength).toBe(message.bodyLength); + expect(index++).toBeLessThan(numMessages); + } + expect(index).toBe(numMessages); + reader.return(); +} + +export async function validateAsyncMessageReader(numMessages: number, source: any) { + const reader = new AsyncMessageReader(source); + let index = 0; + for await (let message of reader) { + + if (index === 0) { + expect(message.isSchema()).toBe(true); + expect(message.bodyLength).toBe(0); + } else { + expect(message.isSchema()).toBe(false); + expect(message.isRecordBatch() || message.isDictionaryBatch()).toBe(true); + } + + try { + expect(message.bodyLength % 8).toBe(0); + } catch (e) { throw new Error(`bodyLength: ${e}`); } + + const body = await reader.readMessageBody(message.bodyLength); + expect(body).toBeInstanceOf(Uint8Array); + expect(body.byteLength).toBe(message.bodyLength); + expect(index++).toBeLessThan(numMessages); + } + expect(index).toBe(numMessages); + await reader.return(); +} diff --git a/src/arrow/js/test/unit/ipc/reader/file-reader-tests.ts b/src/arrow/js/test/unit/ipc/reader/file-reader-tests.ts new file mode 100644 index 000000000..a7ddfc940 --- /dev/null +++ b/src/arrow/js/test/unit/ipc/reader/file-reader-tests.ts @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { + generateRandomTables, + // generateDictionaryTables +} from '../../../data/tables'; +import { ArrowIOTestHelper } from '../helpers'; +import { toArray } from 'ix/asynciterable/toarray'; + +import { + validateRecordBatchReader, + validateAsyncRecordBatchReader +} from '../validate'; + +import { + RecordBatchReader, + RecordBatchFileReader, + AsyncRecordBatchFileReader +} from 'apache-arrow'; + +for (const table of generateRandomTables([10, 20, 30])) { + + const io = ArrowIOTestHelper.file(table); + const name = `[\n ${table.schema.fields.join(',\n ')}\n]`; + + const validate = (source: any) => { validateRecordBatchReader('file', 3, RecordBatchReader.from(source)); }; + const validateAsync = async (source: any) => { await validateAsyncRecordBatchReader('file', 3, await RecordBatchReader.from(source)); }; + const validateAsyncWrapped = async (source: any) => { await validateAsyncRecordBatchReader('file', 3, await RecordBatchReader.from(Promise.resolve(source))); }; + + describe(`RecordBatchFileReader (${name})`, () => { + describe(`should read all RecordBatches`, () => { + test(`Uint8Array`, io.buffer(validate)); + test(`Iterable`, io.iterable(validate)); + }); + describe(`should allow random access to record batches after iterating when autoDestroy=false`, () => { + test(`Uint8Array`, io.buffer(validateRandomAccess)); + test(`Iterable`, io.iterable(validateRandomAccess)); + }); + }); + + describe(`AsyncRecordBatchFileReader (${name})`, () => { + describe(`should read all RecordBatches`, () => { + + test('AsyncIterable', io.asyncIterable(validateAsync)); + test('fs.FileHandle', io.fsFileHandle(validateAsync)); + test('fs.ReadStream', io.fsReadableStream(validateAsync)); + test('stream.Readable', io.nodeReadableStream(validateAsync)); + test('whatwg.ReadableStream', io.whatwgReadableStream(validateAsync)); + test('whatwg.ReadableByteStream', io.whatwgReadableByteStream(validateAsync)); + + test('Promise<AsyncIterable>', io.asyncIterable(validateAsyncWrapped)); + test('Promise<fs.FileHandle>', io.fsFileHandle(validateAsyncWrapped)); + test('Promise<fs.ReadStream>', io.fsReadableStream(validateAsyncWrapped)); + test('Promise<stream.Readable>', io.nodeReadableStream(validateAsyncWrapped)); + test('Promise<ReadableStream>', io.whatwgReadableStream(validateAsyncWrapped)); + test('Promise<ReadableByteStream>', io.whatwgReadableByteStream(validateAsyncWrapped)); + }); + + describe(`should allow random access to record batches after iterating when autoDestroy=false`, () => { + + test('AsyncIterable', io.asyncIterable(validateRandomAccessAsync)); + test('fs.FileHandle', io.fsFileHandle(validateRandomAccessAsync)); + test('fs.ReadStream', io.fsReadableStream(validateRandomAccessAsync)); + test('stream.Readable', io.nodeReadableStream(validateRandomAccessAsync)); + test('whatwg.ReadableStream', io.whatwgReadableStream(validateRandomAccessAsync)); + test('whatwg.ReadableByteStream', io.whatwgReadableByteStream(validateRandomAccessAsync)); + + test('Promise<AsyncIterable>', io.asyncIterable(validateRandomAccessAsync)); + test('Promise<fs.FileHandle>', io.fsFileHandle(validateRandomAccessAsync)); + test('Promise<fs.ReadStream>', io.fsReadableStream(validateRandomAccessAsync)); + test('Promise<stream.Readable>', io.nodeReadableStream(validateRandomAccessAsync)); + test('Promise<ReadableStream>', io.whatwgReadableStream(validateRandomAccessAsync)); + test('Promise<ReadableByteStream>', io.whatwgReadableByteStream(validateRandomAccessAsync)); + }); + }); +} + +function validateRandomAccess(source: any) { + const reader = RecordBatchReader.from(source) as RecordBatchFileReader; + const schema = reader.open({ autoDestroy: false }).schema; + const batches = [...reader]; + expect(reader.closed).toBe(false); + expect(reader.schema).toBe(schema); + while (batches.length > 0) { + const expected = batches.pop()!; + const actual = reader.readRecordBatch(batches.length); + expect(actual).toEqualRecordBatch(expected); + } + reader.cancel(); + expect(reader.closed).toBe(true); + expect(reader.schema).toBeUndefined(); +} + +async function validateRandomAccessAsync(source: any) { + const reader = (await RecordBatchReader.from(source)) as AsyncRecordBatchFileReader; + const schema = (await reader.open({ autoDestroy: false })).schema; + const batches = await toArray(reader); + expect(reader.closed).toBe(false); + expect(reader.schema).toBe(schema); + while (batches.length > 0) { + const expected = batches.pop()!; + const actual = await reader.readRecordBatch(batches.length); + expect(actual).toEqualRecordBatch(expected); + } + await reader.cancel(); + expect(reader.closed).toBe(true); + expect(reader.schema).toBeUndefined(); +} diff --git a/src/arrow/js/test/unit/ipc/reader/from-inference-tests.ts b/src/arrow/js/test/unit/ipc/reader/from-inference-tests.ts new file mode 100644 index 000000000..c444b78fc --- /dev/null +++ b/src/arrow/js/test/unit/ipc/reader/from-inference-tests.ts @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { + generateRandomTables, + // generateDictionaryTables +} from '../../../data/tables'; + +import { ArrowIOTestHelper } from '../helpers'; +import { + RecordBatchReader, + RecordBatchFileReader, + RecordBatchStreamReader, + AsyncRecordBatchFileReader, + AsyncRecordBatchStreamReader +} from 'apache-arrow'; + +for (const table of generateRandomTables([10, 20, 30])) { + const name = `[\n ${table.schema.fields.join(',\n ')}\n]`; + // eslint-disable-next-line jest/valid-describe + describe('RecordBatchReader.from', ((table, name) => () => { + testFromFile(ArrowIOTestHelper.file(table), name); + testFromJSON(ArrowIOTestHelper.json(table), name); + testFromStream(ArrowIOTestHelper.stream(table), name); + })(table, name)); +} + +function testFromJSON(io: ArrowIOTestHelper, name: string) { + describe(`should return a RecordBatchJSONReader (${name})`, () => { + test(`Uint8Array`, io.buffer((buffer) => { + const json = JSON.parse(`${Buffer.from(buffer)}`); + const reader = RecordBatchReader.from(json); + expect(reader.isSync()).toEqual(true); + expect(reader.isAsync()).toEqual(false); + expect(reader).toBeInstanceOf(RecordBatchStreamReader); + })); + }); +} + +function testFromFile(io: ArrowIOTestHelper, name: string) { + + describe(`should return a RecordBatchFileReader (${name})`, () => { + + test(`Uint8Array`, io.buffer(syncSync)); + test(`Iterable`, io.iterable(syncSync)); + test('AsyncIterable', io.asyncIterable(asyncSync)); + test('fs.FileHandle', io.fsFileHandle(asyncAsync)); + test('fs.ReadStream', io.fsReadableStream(asyncSync)); + test('stream.Readable', io.nodeReadableStream(asyncSync)); + test('whatwg.ReadableStream', io.whatwgReadableStream(asyncSync)); + test('whatwg.ReadableByteStream', io.whatwgReadableByteStream(asyncSync)); + + test(`Promise<Uint8Array>`, io.buffer((source) => asyncSync(Promise.resolve(source)))); + test(`Promise<Iterable>`, io.iterable((source) => asyncSync(Promise.resolve(source)))); + test('Promise<AsyncIterable>', io.asyncIterable((source) => asyncSync(Promise.resolve(source)))); + test('Promise<fs.FileHandle>', io.fsFileHandle((source) => asyncAsync(Promise.resolve(source)))); + test('Promise<fs.ReadStream>', io.fsReadableStream((source) => asyncSync(Promise.resolve(source)))); + test('Promise<stream.Readable>', io.nodeReadableStream((source) => asyncSync(Promise.resolve(source)))); + test('Promise<whatwg.ReadableStream>', io.whatwgReadableStream((source) => asyncSync(Promise.resolve(source)))); + test('Promise<whatwg.ReadableByteStream>', io.whatwgReadableByteStream((source) => asyncSync(Promise.resolve(source)))); + }); + + function syncSync(source: any) { + const reader = RecordBatchReader.from(source); + expect(reader.isSync()).toEqual(true); + expect(reader.isAsync()).toEqual(false); + expect(reader).toBeInstanceOf(RecordBatchFileReader); + } + + async function asyncSync(source: any) { + const pending = RecordBatchReader.from(source); + expect(pending).toBeInstanceOf(Promise); + const reader = await pending; + expect(reader.isSync()).toEqual(true); + expect(reader.isAsync()).toEqual(false); + expect(reader).toBeInstanceOf(RecordBatchFileReader); + } + + async function asyncAsync(source: any) { + const pending = RecordBatchReader.from(source); + expect(pending).toBeInstanceOf(Promise); + const reader = await pending; + expect(reader.isSync()).toEqual(false); + expect(reader.isAsync()).toEqual(true); + expect(reader).toBeInstanceOf(AsyncRecordBatchFileReader); + } +} + +function testFromStream(io: ArrowIOTestHelper, name: string) { + + describe(`should return a RecordBatchStreamReader (${name})`, () => { + + test(`Uint8Array`, io.buffer(syncSync)); + test(`Iterable`, io.iterable(syncSync)); + test('AsyncIterable', io.asyncIterable(asyncAsync)); + test('fs.FileHandle', io.fsFileHandle(asyncAsync)); + test('fs.ReadStream', io.fsReadableStream(asyncAsync)); + test('stream.Readable', io.nodeReadableStream(asyncAsync)); + test('whatwg.ReadableStream', io.whatwgReadableStream(asyncAsync)); + test('whatwg.ReadableByteStream', io.whatwgReadableByteStream(asyncAsync)); + + test(`Promise<Uint8Array>`, io.buffer((source) => asyncSync(Promise.resolve(source)))); + test(`Promise<Iterable>`, io.iterable((source) => asyncSync(Promise.resolve(source)))); + test('Promise<AsyncIterable>', io.asyncIterable((source) => asyncAsync(Promise.resolve(source)))); + test('Promise<fs.FileHandle>', io.fsFileHandle((source) => asyncAsync(Promise.resolve(source)))); + test('Promise<fs.ReadStream>', io.fsReadableStream((source) => asyncAsync(Promise.resolve(source)))); + test('Promise<stream.Readable>', io.nodeReadableStream((source) => asyncAsync(Promise.resolve(source)))); + test('Promise<whatwg.ReadableStream>', io.whatwgReadableStream((source) => asyncAsync(Promise.resolve(source)))); + test('Promise<whatwg.ReadableByteStream>', io.whatwgReadableByteStream((source) => asyncAsync(Promise.resolve(source)))); + }); + + function syncSync(source: any) { + const reader = RecordBatchReader.from(source); + expect(reader.isSync()).toEqual(true); + expect(reader.isAsync()).toEqual(false); + expect(reader).toBeInstanceOf(RecordBatchStreamReader); + } + + async function asyncSync(source: any) { + const pending = RecordBatchReader.from(source); + expect(pending).toBeInstanceOf(Promise); + const reader = await pending; + expect(reader.isSync()).toEqual(true); + expect(reader.isAsync()).toEqual(false); + expect(reader).toBeInstanceOf(RecordBatchStreamReader); + } + + async function asyncAsync(source: any) { + const pending = RecordBatchReader.from(source); + expect(pending).toBeInstanceOf(Promise); + const reader = await pending; + expect(reader.isSync()).toEqual(false); + expect(reader.isAsync()).toEqual(true); + expect(reader).toBeInstanceOf(AsyncRecordBatchStreamReader); + } +} diff --git a/src/arrow/js/test/unit/ipc/reader/json-reader-tests.ts b/src/arrow/js/test/unit/ipc/reader/json-reader-tests.ts new file mode 100644 index 000000000..9bd1e3466 --- /dev/null +++ b/src/arrow/js/test/unit/ipc/reader/json-reader-tests.ts @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { + generateRandomTables, + // generateDictionaryTables +} from '../../../data/tables'; + +import { ArrowIOTestHelper } from '../helpers'; +import { RecordBatchReader } from 'apache-arrow'; +import { validateRecordBatchReader } from '../validate'; + +for (const table of generateRandomTables([10, 20, 30])) { + + const io = ArrowIOTestHelper.json(table); + const name = `[\n ${table.schema.fields.join(',\n ')}\n]`; + + describe(`RecordBatchJSONReader (${name})`, () => { + describe(`should read all RecordBatches`, () => { + test(`Uint8Array`, io.buffer((buffer) => { + const json = JSON.parse(Buffer.from(buffer).toString()); + validateRecordBatchReader('json', 3, RecordBatchReader.from(json)); + })); + }); + }); +} diff --git a/src/arrow/js/test/unit/ipc/reader/stream-reader-tests.ts b/src/arrow/js/test/unit/ipc/reader/stream-reader-tests.ts new file mode 100644 index 000000000..23879cf79 --- /dev/null +++ b/src/arrow/js/test/unit/ipc/reader/stream-reader-tests.ts @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { + generateRandomTables, + // generateDictionaryTables +} from '../../../data/tables'; + +import { + validateRecordBatchReader, + validateAsyncRecordBatchReader +} from '../validate'; + +import { ArrowIOTestHelper } from '../helpers'; +import { RecordBatchReader } from 'apache-arrow'; + +for (const table of generateRandomTables([10, 20, 30])) { + + const io = ArrowIOTestHelper.stream(table); + const name = `[\n ${table.schema.fields.join(',\n ')}\n]`; + + const validate = (source: any) => { validateRecordBatchReader('stream', 3, RecordBatchReader.from(source)); }; + const validateAsync = async (source: any) => { await validateAsyncRecordBatchReader('stream', 3, await RecordBatchReader.from(source)); }; + const validateAsyncWrapped = async (source: any) => { await validateAsyncRecordBatchReader('stream', 3, await RecordBatchReader.from(Promise.resolve(source))); }; + + describe(`RecordBatchStreamReader (${name})`, () => { + describe(`should read all RecordBatches`, () => { + test(`Uint8Array`, io.buffer(validate)); + test(`Iterable`, io.iterable(validate)); + }); + }); + + describe(`AsyncRecordBatchStreamReader (${name})`, () => { + describe(`should read all RecordBatches`, () => { + + test('AsyncIterable', io.asyncIterable(validateAsync)); + test('fs.FileHandle', io.fsFileHandle(validateAsync)); + test('fs.ReadStream', io.fsReadableStream(validateAsync)); + test('stream.Readable', io.nodeReadableStream(validateAsync)); + test('whatwg.ReadableStream', io.whatwgReadableStream(validateAsync)); + test('whatwg.ReadableByteStream', io.whatwgReadableByteStream(validateAsync)); + + test('Promise<AsyncIterable>', io.asyncIterable(validateAsyncWrapped)); + test('Promise<fs.FileHandle>', io.fsFileHandle(validateAsyncWrapped)); + test('Promise<fs.ReadStream>', io.fsReadableStream(validateAsyncWrapped)); + test('Promise<stream.Readable>', io.nodeReadableStream(validateAsyncWrapped)); + test('Promise<ReadableStream>', io.whatwgReadableStream(validateAsyncWrapped)); + test('Promise<ReadableByteStream>', io.whatwgReadableByteStream(validateAsyncWrapped)); + }); + }); +} diff --git a/src/arrow/js/test/unit/ipc/reader/streams-dom-tests.ts b/src/arrow/js/test/unit/ipc/reader/streams-dom-tests.ts new file mode 100644 index 000000000..a380e1619 --- /dev/null +++ b/src/arrow/js/test/unit/ipc/reader/streams-dom-tests.ts @@ -0,0 +1,227 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { + generateRandomTables, + // generateDictionaryTables +} from '../../../data/tables'; + +import { + Table, + RecordBatchReader, + RecordBatchStreamWriter +} from 'apache-arrow'; + +import { validateRecordBatchAsyncIterator } from '../validate'; +import { ArrowIOTestHelper, readableDOMStreamToAsyncIterator } from '../helpers'; + +(() => { + + if (process.env.TEST_DOM_STREAMS !== 'true') { + return test('not testing DOM streams because process.env.TEST_DOM_STREAMS !== "true"', () => {}); + } + + for (const table of generateRandomTables([10, 20, 30])) { + + const file = ArrowIOTestHelper.file(table); + const json = ArrowIOTestHelper.json(table); + const stream = ArrowIOTestHelper.stream(table); + const name = `[\n ${table.schema.fields.join(',\n ')}\n]`; + + describe(`RecordBatchReader.throughDOM (${name})`, () => { + describe('file', () => { + test('ReadableStream', file.whatwgReadableStream(validate)); + test('ReadableByteStream', file.whatwgReadableByteStream(validate)); + }); + describe('stream', () => { + test('ReadableStream', stream.whatwgReadableStream(validate)); + test('ReadableByteStream', stream.whatwgReadableByteStream(validate)); + }); + async function validate(source: ReadableStream) { + const stream = source.pipeThrough(RecordBatchReader.throughDOM()); + await validateRecordBatchAsyncIterator(3, readableDOMStreamToAsyncIterator(stream)); + } + }); + + describe(`toDOMStream (${name})`, () => { + + describe(`RecordBatchJSONReader`, () => { + test('Uint8Array', json.buffer((source) => validate(JSON.parse(`${Buffer.from(source)}`)))); + }); + + describe(`RecordBatchFileReader`, () => { + test(`Uint8Array`, file.buffer(validate)); + test(`Iterable`, file.iterable(validate)); + test('AsyncIterable', file.asyncIterable(validate)); + test('fs.FileHandle', file.fsFileHandle(validate)); + test('fs.ReadStream', file.fsReadableStream(validate)); + test('stream.Readable', file.nodeReadableStream(validate)); + test('whatwg.ReadableStream', file.whatwgReadableStream(validate)); + test('whatwg.ReadableByteStream', file.whatwgReadableByteStream(validate)); + test('Promise<AsyncIterable>', file.asyncIterable((source) => validate(Promise.resolve(source)))); + test('Promise<fs.FileHandle>', file.fsFileHandle((source) => validate(Promise.resolve(source)))); + test('Promise<fs.ReadStream>', file.fsReadableStream((source) => validate(Promise.resolve(source)))); + test('Promise<stream.Readable>', file.nodeReadableStream((source) => validate(Promise.resolve(source)))); + test('Promise<ReadableStream>', file.whatwgReadableStream((source) => validate(Promise.resolve(source)))); + test('Promise<ReadableByteStream>', file.whatwgReadableByteStream((source) => validate(Promise.resolve(source)))); + }); + + describe(`RecordBatchStreamReader`, () => { + test(`Uint8Array`, stream.buffer(validate)); + test(`Iterable`, stream.iterable(validate)); + test('AsyncIterable', stream.asyncIterable(validate)); + test('fs.FileHandle', stream.fsFileHandle(validate)); + test('fs.ReadStream', stream.fsReadableStream(validate)); + test('stream.Readable', stream.nodeReadableStream(validate)); + test('whatwg.ReadableStream', stream.whatwgReadableStream(validate)); + test('whatwg.ReadableByteStream', stream.whatwgReadableByteStream(validate)); + test('Promise<AsyncIterable>', stream.asyncIterable((source) => validate(Promise.resolve(source)))); + test('Promise<fs.FileHandle>', stream.fsFileHandle((source) => validate(Promise.resolve(source)))); + test('Promise<fs.ReadStream>', stream.fsReadableStream((source) => validate(Promise.resolve(source)))); + test('Promise<stream.Readable>', stream.nodeReadableStream((source) => validate(Promise.resolve(source)))); + test('Promise<ReadableStream>', stream.whatwgReadableStream((source) => validate(Promise.resolve(source)))); + test('Promise<ReadableByteStream>', stream.whatwgReadableByteStream((source) => validate(Promise.resolve(source)))); + }); + + async function validate(source: any) { + const reader: RecordBatchReader = await RecordBatchReader.from(source); + const iterator = readableDOMStreamToAsyncIterator(reader.toDOMStream()); + await validateRecordBatchAsyncIterator(3, iterator); + } + }); + } + + it('readAll() should pipe to separate WhatWG WritableStreams', async () => { + // @ts-ignore + const { concatStream } = await import('@openpgp/web-stream-tools'); + + expect.hasAssertions(); + + const tables = [...generateRandomTables([10, 20, 30])]; + + const stream = concatStream(tables.map((table, i) => + RecordBatchStreamWriter.writeAll(table).toDOMStream({ + // Alternate between bytes mode and regular mode because code coverage + type: i % 2 === 0 ? 'bytes' : undefined + }) + )) as ReadableStream<Uint8Array>; + + let tableIndex = -1; + let reader: RecordBatchReader | undefined; + + for await (reader of RecordBatchReader.readAll(stream)) { + + validateStreamState(reader, stream, false); + + const output = reader + .pipeThrough(RecordBatchStreamWriter.throughDOM()) + .pipeThrough(new TransformStream()); + + validateStreamState(reader, output, false, false); + + const sourceTable = tables[++tableIndex]; + const streamTable = await Table.from(output); + expect(streamTable).toEqualTable(sourceTable); + expect(output.locked).toBe(false); + } + + expect(reader).toBeDefined(); + validateStreamState(reader!, stream, true); + expect(tableIndex).toBe(tables.length - 1); + }); + + it('should not close the underlying WhatWG ReadableStream when reading multiple tables to completion', async () => { + // @ts-ignore + const { concatStream } = await import('@openpgp/web-stream-tools'); + + expect.hasAssertions(); + + const tables = [...generateRandomTables([10, 20, 30])]; + + const stream = concatStream(tables.map((table, i) => + RecordBatchStreamWriter.writeAll(table).toDOMStream({ + // Alternate between bytes mode and regular mode because code coverage + type: i % 2 === 0 ? 'bytes' : undefined + }) + )) as ReadableStream<Uint8Array>; + + let tableIndex = -1; + let reader = await RecordBatchReader.from(stream); + + validateStreamState(reader, stream, false); + + for await (reader of RecordBatchReader.readAll(reader)) { + + validateStreamState(reader, stream, false); + + const sourceTable = tables[++tableIndex]; + const streamTable = await Table.from(reader); + expect(streamTable).toEqualTable(sourceTable); + } + + validateStreamState(reader, stream, true); + expect(tableIndex).toBe(tables.length - 1); + }); + + it('should close the underlying WhatWG ReadableStream when reading multiple tables and we break early', async () => { + // @ts-ignore + const { concatStream } = await import('@openpgp/web-stream-tools'); + + expect.hasAssertions(); + + const tables = [...generateRandomTables([10, 20, 30])]; + + const stream = concatStream(tables.map((table, i) => + RecordBatchStreamWriter.writeAll(table).toDOMStream({ + // Alternate between bytes mode and regular mode because code coverage + type: i % 2 === 0 ? 'bytes' : undefined + }) + )) as ReadableStream<Uint8Array>; + + let tableIndex = -1; + let reader = await RecordBatchReader.from(stream); + + validateStreamState(reader, stream, false); + + for await (reader of RecordBatchReader.readAll(reader)) { + + validateStreamState(reader, stream, false); + + let batchIndex = -1; + const sourceTable = tables[++tableIndex]; + const breakEarly = tableIndex === (tables.length / 2 | 0); + + for await (const streamBatch of reader) { + expect(streamBatch).toEqualRecordBatch(sourceTable.chunks[++batchIndex]); + if (breakEarly && batchIndex === 1) { break; } + } + if (breakEarly) { + // the reader should stay open until we break from the outermost loop + validateStreamState(reader, stream, false); + break; + } + } + + validateStreamState(reader, stream, true); + expect(tableIndex).toBe(tables.length / 2 | 0); + }); +})(); + +function validateStreamState(reader: RecordBatchReader, stream: ReadableStream, closed: boolean, locked = !closed) { + expect(reader.closed).toBe(closed); + expect(stream.locked).toBe(locked); +} diff --git a/src/arrow/js/test/unit/ipc/reader/streams-node-tests.ts b/src/arrow/js/test/unit/ipc/reader/streams-node-tests.ts new file mode 100644 index 000000000..822f99350 --- /dev/null +++ b/src/arrow/js/test/unit/ipc/reader/streams-node-tests.ts @@ -0,0 +1,219 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { + generateRandomTables +} from '../../../data/tables'; + +import { + Table, + RecordBatchReader, + RecordBatchStreamWriter +} from 'apache-arrow'; + +import { ArrowIOTestHelper } from '../helpers'; +import { validateRecordBatchAsyncIterator } from '../validate'; + +(() => { + + if (process.env.TEST_NODE_STREAMS !== 'true') { + return test('not testing node streams because process.env.TEST_NODE_STREAMS !== "true"', () => {}); + } + + for (const table of generateRandomTables([10, 20, 30])) { + + const file = ArrowIOTestHelper.file(table); + const json = ArrowIOTestHelper.json(table); + const stream = ArrowIOTestHelper.stream(table); + const name = `[\n ${table.schema.fields.join(',\n ')}\n]`; + + describe(`RecordBatchReader.throughNode (${name})`, () => { + describe('file', () => { + test('fs.ReadStream', file.fsReadableStream(validate)); + test('stream.Readable', file.nodeReadableStream(validate)); + }); + describe('stream', () => { + test('fs.ReadStream', file.fsReadableStream(validate)); + test('stream.Readable', file.nodeReadableStream(validate)); + }); + async function validate(source: NodeJS.ReadableStream) { + const stream = source.pipe(RecordBatchReader.throughNode()); + await validateRecordBatchAsyncIterator(3, stream[Symbol.asyncIterator]()); + } + }); + + describe(`toNodeStream (${name})`, () => { + + describe(`RecordBatchJSONReader`, () => { + test('Uint8Array', json.buffer((source) => validate(JSON.parse(`${Buffer.from(source)}`)))); + }); + + describe(`RecordBatchFileReader`, () => { + test(`Uint8Array`, file.buffer(validate)); + test(`Iterable`, file.iterable(validate)); + test('AsyncIterable', file.asyncIterable(validate)); + test('fs.FileHandle', file.fsFileHandle(validate)); + test('fs.ReadStream', file.fsReadableStream(validate)); + test('stream.Readable', file.nodeReadableStream(validate)); + test('whatwg.ReadableStream', file.whatwgReadableStream(validate)); + test('whatwg.ReadableByteStream', file.whatwgReadableByteStream(validate)); + test('Promise<AsyncIterable>', file.asyncIterable((source) => validate(Promise.resolve(source)))); + test('Promise<fs.FileHandle>', file.fsFileHandle((source) => validate(Promise.resolve(source)))); + test('Promise<fs.ReadStream>', file.fsReadableStream((source) => validate(Promise.resolve(source)))); + test('Promise<stream.Readable>', file.nodeReadableStream((source) => validate(Promise.resolve(source)))); + test('Promise<ReadableStream>', file.whatwgReadableStream((source) => validate(Promise.resolve(source)))); + test('Promise<ReadableByteStream>', file.whatwgReadableByteStream((source) => validate(Promise.resolve(source)))); + }); + + describe(`RecordBatchStreamReader`, () => { + test(`Uint8Array`, stream.buffer(validate)); + test(`Iterable`, stream.iterable(validate)); + test('AsyncIterable', stream.asyncIterable(validate)); + test('fs.FileHandle', stream.fsFileHandle(validate)); + test('fs.ReadStream', stream.fsReadableStream(validate)); + test('stream.Readable', stream.nodeReadableStream(validate)); + test('whatwg.ReadableStream', stream.whatwgReadableStream(validate)); + test('whatwg.ReadableByteStream', stream.whatwgReadableByteStream(validate)); + test('Promise<AsyncIterable>', stream.asyncIterable((source) => validate(Promise.resolve(source)))); + test('Promise<fs.FileHandle>', stream.fsFileHandle((source) => validate(Promise.resolve(source)))); + test('Promise<fs.ReadStream>', stream.fsReadableStream((source) => validate(Promise.resolve(source)))); + test('Promise<stream.Readable>', stream.nodeReadableStream((source) => validate(Promise.resolve(source)))); + test('Promise<ReadableStream>', stream.whatwgReadableStream((source) => validate(Promise.resolve(source)))); + test('Promise<ReadableByteStream>', stream.whatwgReadableByteStream((source) => validate(Promise.resolve(source)))); + }); + + async function validate(source: any) { + const reader: RecordBatchReader = await RecordBatchReader.from(source); + await validateRecordBatchAsyncIterator(3, reader.toNodeStream()[Symbol.asyncIterator]()); + } + }); + } + + it('readAll() should pipe to separate NodeJS WritableStreams', async () => { + // @ts-ignore + const { default: MultiStream } = await import('multistream'); + const { PassThrough } = await import('stream'); + + expect.hasAssertions(); + + const tables = [...generateRandomTables([10, 20, 30])]; + + const stream = new MultiStream(tables.map((table) => + () => RecordBatchStreamWriter.writeAll(table).toNodeStream() + )) as NodeJS.ReadableStream; + + let tableIndex = -1; + let reader: RecordBatchReader | undefined; + + for await (reader of RecordBatchReader.readAll(stream)) { + + validateStreamState(reader, stream, false); + + const output = reader + .pipe(RecordBatchStreamWriter.throughNode()) + .pipe(new PassThrough()); + + validateStreamState(reader, output, false); + + const sourceTable = tables[++tableIndex]; + const streamTable = await Table.from(output); + expect(streamTable).toEqualTable(sourceTable); + expect(Boolean(output.readableFlowing)).toBe(false); + } + + expect(reader).toBeDefined(); + validateStreamState(reader!, stream, true); + expect(tableIndex).toBe(tables.length - 1); + }); + + it('should not close the underlying NodeJS ReadableStream when reading multiple tables to completion', async () => { + // @ts-ignore + const { default: MultiStream } = await import('multistream'); + + expect.hasAssertions(); + + const tables = [...generateRandomTables([10, 20, 30])]; + + const stream = new MultiStream(tables.map((table) => + () => RecordBatchStreamWriter.writeAll(table).toNodeStream() + )) as NodeJS.ReadableStream; + + let tableIndex = -1; + let reader = await RecordBatchReader.from(stream); + + validateStreamState(reader, stream, false); + + for await (reader of RecordBatchReader.readAll(reader)) { + + validateStreamState(reader, stream, false); + + const sourceTable = tables[++tableIndex]; + const streamTable = await Table.from(reader); + expect(streamTable).toEqualTable(sourceTable); + } + + validateStreamState(reader, stream, true); + expect(tableIndex).toBe(tables.length - 1); + }); + + it('should close the underlying NodeJS ReadableStream when reading multiple tables and we break early', async () => { + // @ts-ignore + const { default: MultiStream } = await import('multistream'); + + expect.hasAssertions(); + + const tables = [...generateRandomTables([10, 20, 30])]; + + const stream = new MultiStream(tables.map((table) => + () => RecordBatchStreamWriter.writeAll(table).toNodeStream() + )) as NodeJS.ReadableStream; + + let tableIndex = -1; + let reader = await RecordBatchReader.from(stream); + + validateStreamState(reader, stream, false); + + for await (reader of RecordBatchReader.readAll(reader)) { + + validateStreamState(reader, stream, false); + + let batchIndex = -1; + const sourceTable = tables[++tableIndex]; + const breakEarly = tableIndex === (tables.length / 2 | 0); + + for await (const streamBatch of reader) { + expect(streamBatch).toEqualRecordBatch(sourceTable.chunks[++batchIndex]); + if (breakEarly && batchIndex === 1) { break; } + } + if (breakEarly) { + // the reader should stay open until we break from the outermost loop + validateStreamState(reader, stream, false); + break; + } + } + + validateStreamState(reader, stream, true, true); + expect(tableIndex).toBe(tables.length / 2 | 0); + }); +})(); + +function validateStreamState(reader: RecordBatchReader, stream: NodeJS.ReadableStream, closed: boolean, readable = !closed) { + expect(reader.closed).toBe(closed); + expect(Boolean(stream.readable)).toBe(readable); + expect(Boolean((stream as any).destroyed)).toBe(closed); + expect(Boolean((stream as any).readableFlowing)).toBe(false); +} diff --git a/src/arrow/js/test/unit/ipc/validate.ts b/src/arrow/js/test/unit/ipc/validate.ts new file mode 100644 index 000000000..aedf87a2d --- /dev/null +++ b/src/arrow/js/test/unit/ipc/validate.ts @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import '../../jest-extensions'; + +import { + Schema, + RecordBatch, + RecordBatchReader, + RecordBatchFileReader, + RecordBatchStreamReader, +} from 'apache-arrow'; + +export function validateRecordBatchReader<T extends RecordBatchFileReader | RecordBatchStreamReader>(type: 'json' | 'file' | 'stream', numBatches: number, r: T) { + const reader = r.open(); + expect(reader).toBeInstanceOf(RecordBatchReader); + expect(type === 'file' ? reader.isFile() : reader.isStream()).toBe(true); + expect(reader.schema).toBeInstanceOf(Schema); + validateRecordBatchIterator(numBatches, reader[Symbol.iterator]()); + expect(reader.closed).toBe(reader.autoDestroy); + return reader; +} + +export async function validateAsyncRecordBatchReader<T extends RecordBatchReader>(type: 'json' | 'file' | 'stream', numBatches: number, r: T) { + const reader = await r.open(); + expect(reader).toBeInstanceOf(RecordBatchReader); + expect(reader.schema).toBeInstanceOf(Schema); + expect(type === 'file' ? reader.isFile() : reader.isStream()).toBe(true); + await validateRecordBatchAsyncIterator(numBatches, reader[Symbol.asyncIterator]()); + expect(reader.closed).toBe(reader.autoDestroy); + return reader; +} + +export function validateRecordBatchIterator(numBatches: number, iterator: Iterable<RecordBatch> | IterableIterator<RecordBatch>) { + let i = 0; + try { + for (const recordBatch of iterator) { + expect(recordBatch).toBeInstanceOf(RecordBatch); + expect(i++).toBeLessThan(numBatches); + } + } catch (e) { throw new Error(`${i}: ${e}`); } + expect(i).toBe(numBatches); + if (typeof (iterator as any).return === 'function') { + (iterator as any).return(); + } +} + +export async function validateRecordBatchAsyncIterator(numBatches: number, iterator: AsyncIterable<RecordBatch> | AsyncIterableIterator<RecordBatch>) { + let i = 0; + try { + for await (const recordBatch of iterator) { + expect(recordBatch).toBeInstanceOf(RecordBatch); + expect(i++).toBeLessThan(numBatches); + } + } catch (e) { throw new Error(`${i}: ${e}`); } + expect(i).toBe(numBatches); + if (typeof (iterator as any).return === 'function') { + await (iterator as any).return(); + } +} diff --git a/src/arrow/js/test/unit/ipc/writer/file-writer-tests.ts b/src/arrow/js/test/unit/ipc/writer/file-writer-tests.ts new file mode 100644 index 000000000..fa639e5f6 --- /dev/null +++ b/src/arrow/js/test/unit/ipc/writer/file-writer-tests.ts @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { + generateRandomTables, + generateDictionaryTables +} from '../../../data/tables'; + +import { validateRecordBatchIterator } from '../validate'; +import { Table, RecordBatchFileWriter } from 'apache-arrow'; + +describe('RecordBatchFileWriter', () => { + for (const table of generateRandomTables([10, 20, 30])) { + testFileWriter(table, `[${table.schema.fields.join(', ')}]`); + } + for (const table of generateDictionaryTables([10, 20, 30])) { + testFileWriter(table, `${table.schema.fields[0]}`); + } +}); + +function testFileWriter(table: Table, name: string) { + describe(`should write the Arrow IPC file format (${name})`, () => { + test(`Table`, validateTable.bind(0, table)); + }); +} + +async function validateTable(source: Table) { + const writer = RecordBatchFileWriter.writeAll(source); + const result = await Table.from(writer.toUint8Array()); + validateRecordBatchIterator(3, source.chunks); + expect(result).toEqualTable(source); +} diff --git a/src/arrow/js/test/unit/ipc/writer/json-writer-tests.ts b/src/arrow/js/test/unit/ipc/writer/json-writer-tests.ts new file mode 100644 index 000000000..05be0e272 --- /dev/null +++ b/src/arrow/js/test/unit/ipc/writer/json-writer-tests.ts @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { + generateRandomTables, + generateDictionaryTables +} from '../../../data/tables'; + +import { validateRecordBatchIterator } from '../validate'; +import { Table, RecordBatchJSONWriter } from 'apache-arrow'; + +describe('RecordBatchJSONWriter', () => { + for (const table of generateRandomTables([10, 20, 30])) { + testJSONWriter(table, `[${table.schema.fields.join(', ')}]`); + } + for (const table of generateDictionaryTables([10, 20, 30])) { + testJSONWriter(table, `${table.schema.fields[0]}`); + } +}); + +function testJSONWriter(table: Table, name: string) { + describe(`should write the Arrow IPC JSON format (${name})`, () => { + test(`Table`, validateTable.bind(0, table)); + }); +} + +async function validateTable(source: Table) { + const writer = RecordBatchJSONWriter.writeAll(source); + const result = Table.from(JSON.parse(await writer.toString())); + validateRecordBatchIterator(3, source.chunks); + expect(result).toEqualTable(source); +} diff --git a/src/arrow/js/test/unit/ipc/writer/stream-writer-tests.ts b/src/arrow/js/test/unit/ipc/writer/stream-writer-tests.ts new file mode 100644 index 000000000..a83aa39da --- /dev/null +++ b/src/arrow/js/test/unit/ipc/writer/stream-writer-tests.ts @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { + generateRandomTables, + generateDictionaryTables +} from '../../../data/tables'; + +import * as generate from '../../../generate-test-data'; +import { validateRecordBatchIterator } from '../validate'; +import { RecordBatchStreamWriterOptions } from 'apache-arrow/ipc/writer'; +import { DictionaryVector, Dictionary, Uint32, Int32 } from 'apache-arrow'; +import { Table, Schema, Field, Chunked, Builder, RecordBatch, RecordBatchReader, RecordBatchStreamWriter } from 'apache-arrow'; + +describe('RecordBatchStreamWriter', () => { + + (() => { + const type = generate.sparseUnion(0, 0).vector.type; + const schema = new Schema([new Field('dictSparseUnion', type)]); + const table = generate.table([10, 20, 30], schema).table; + const testName = `[${table.schema.fields.join(', ')}]`; + testStreamWriter(table, testName, { writeLegacyIpcFormat: true }); + testStreamWriter(table, testName, { writeLegacyIpcFormat: false }); + })(); + + for (const table of generateRandomTables([10, 20, 30])) { + const testName = `[${table.schema.fields.join(', ')}]`; + testStreamWriter(table, testName, { writeLegacyIpcFormat: true }); + testStreamWriter(table, testName, { writeLegacyIpcFormat: false }); + } + + for (const table of generateDictionaryTables([10, 20, 30])) { + const testName = `${table.schema.fields[0]}`; + testStreamWriter(table, testName, { writeLegacyIpcFormat: true }); + testStreamWriter(table, testName, { writeLegacyIpcFormat: false }); + } + + it(`should write multiple tables to the same output stream`, async () => { + const tables = [] as Table[]; + const writer = new RecordBatchStreamWriter({ autoDestroy: false }); + const validate = (async () => { + for await (const reader of RecordBatchReader.readAll(writer)) { + const sourceTable = tables.shift()!; + const streamTable = await Table.from(reader); + expect(streamTable).toEqualTable(sourceTable); + } + })(); + for (const table of generateRandomTables([10, 20, 30])) { + tables.push(table); + await writer.writeAll((async function*() { + for (const chunk of table.chunks) { + yield chunk; // insert some asynchrony + await new Promise((r) => setTimeout(r, 1)); + } + }())); + } + writer.close(); + await validate; + }); + + it('should write delta dictionary batches', async () => { + + const name = 'dictionary_encoded_uint32'; + const chunks: DictionaryVector<Uint32, Int32>[] = []; + const { + vector: sourceVector, values: sourceValues, + } = generate.dictionary(1000, 20, new Uint32(), new Int32()); + + const writer = RecordBatchStreamWriter.writeAll((function* () { + const transform = Builder.throughIterable({ + type: sourceVector.type, nullValues: [null], + queueingStrategy: 'count', highWaterMark: 50, + }); + for (const chunk of transform(sourceValues())) { + chunks.push(chunk); + yield RecordBatch.new({ [name]: chunk }); + } + })()); + + expect(Chunked.concat(chunks)).toEqualVector(sourceVector); + + type T = { [name]: Dictionary<Uint32, Int32> }; + const sourceTable = Table.new({ [name]: sourceVector }); + const resultTable = await Table.from<T>(writer.toUint8Array()); + + const { dictionary } = resultTable.getColumn(name); + + expect(resultTable).toEqualTable(sourceTable); + expect((dictionary as Chunked)).toBeInstanceOf(Chunked); + expect((dictionary as Chunked).chunks).toHaveLength(20); + }); +}); + +function testStreamWriter(table: Table, name: string, options: RecordBatchStreamWriterOptions) { + describe(`should write the Arrow IPC stream format (${name})`, () => { + test(`Table`, validateTable.bind(0, table, options)); + }); +} + +async function validateTable(source: Table, options: RecordBatchStreamWriterOptions) { + const writer = RecordBatchStreamWriter.writeAll(source, options); + const result = await Table.from(writer.toUint8Array()); + validateRecordBatchIterator(3, source.chunks); + expect(result).toEqualTable(source); +} diff --git a/src/arrow/js/test/unit/ipc/writer/streams-dom-tests.ts b/src/arrow/js/test/unit/ipc/writer/streams-dom-tests.ts new file mode 100644 index 000000000..a19ddcdd7 --- /dev/null +++ b/src/arrow/js/test/unit/ipc/writer/streams-dom-tests.ts @@ -0,0 +1,272 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { + generateRandomTables, + // generateDictionaryTables +} from '../../../data/tables'; + +import { from, as } from 'ix/asynciterable'; +import { tap, flatMap } from 'ix/asynciterable/operators'; + +import { + Table, + RecordBatchReader, + RecordBatchWriter, + RecordBatchFileWriter, + RecordBatchJSONWriter, + RecordBatchStreamWriter, +} from 'apache-arrow'; + +import { + ArrowIOTestHelper, + concatBuffersAsync, + readableDOMStreamToAsyncIterator +} from '../helpers'; + +import { + validateRecordBatchReader, + validateAsyncRecordBatchReader, + validateRecordBatchAsyncIterator +} from '../validate'; + +(() => { + + if (process.env.TEST_DOM_STREAMS !== 'true') { + return test('not testing DOM streams because process.env.TEST_DOM_STREAMS !== "true"', () => {}); + } + + for (const table of generateRandomTables([10, 20, 30])) { + + const file = ArrowIOTestHelper.file(table); + const json = ArrowIOTestHelper.json(table); + const stream = ArrowIOTestHelper.stream(table); + const name = `[\n ${table.schema.fields.join(',\n ')}\n]`; + + describe(`RecordBatchWriter.throughDOM (${name})`, () => { + + describe('file', () => { + describe(`convert`, () => { + test('ReadableStream', file.whatwgReadableStream(validateConvert.bind(0, RecordBatchStreamWriter))); + test('ReadableByteStream', file.whatwgReadableByteStream(validateConvert.bind(0, RecordBatchStreamWriter))); + }); + describe(`through`, () => { + test('ReadableStream', file.whatwgReadableStream(validateThrough.bind(0, RecordBatchFileWriter))); + test('ReadableByteStream', file.whatwgReadableByteStream(validateThrough.bind(0, RecordBatchFileWriter))); + }); + }); + + describe('stream', () => { + describe(`convert`, () => { + test('ReadableStream', stream.whatwgReadableStream(validateConvert.bind(0, RecordBatchFileWriter))); + test('ReadableByteStream', stream.whatwgReadableByteStream(validateConvert.bind(0, RecordBatchFileWriter))); + }); + describe(`through`, () => { + test('ReadableStream', stream.whatwgReadableStream(validateThrough.bind(0, RecordBatchStreamWriter))); + test('ReadableByteStream', stream.whatwgReadableByteStream(validateThrough.bind(0, RecordBatchStreamWriter))); + }); + }); + + async function validateConvert(RBWImplementation: typeof RecordBatchWriter, source: ReadableStream) { + const stream = source + .pipeThrough(RecordBatchReader.throughDOM()) + .pipeThrough(RBWImplementation.throughDOM()); + const type = RBWImplementation === RecordBatchFileWriter ? 'file' : 'stream'; + await validateAsyncRecordBatchReader(type, 3, await RecordBatchReader.from(stream)); + } + + async function validateThrough(RBWImplementation: typeof RecordBatchWriter, source: ReadableStream) { + const stream = source + .pipeThrough(RecordBatchReader.throughDOM()) + .pipeThrough(RBWImplementation.throughDOM()) + .pipeThrough(RecordBatchReader.throughDOM()); + await validateRecordBatchAsyncIterator(3, readableDOMStreamToAsyncIterator(stream)); + } + }); + + describe(`toDOMStream (${name})`, () => { + + const wrapArgInPromise = (fn: (p: Promise<any>) => any) => (x: any) => fn(Promise.resolve(x)); + + describe(`RecordBatchJSONWriter`, () => { + + const toJSON = (x: any): { schema: any } => JSON.parse(`${Buffer.from(x)}`); + + test('Uint8Array', json.buffer((source) => validate(toJSON(source)))); + test('Promise<Uint8Array>', json.buffer((source) => validate(Promise.resolve(toJSON(source))))); + + async function validate(source: { schema: any } | Promise<{ schema: any }>) { + const reader = await RecordBatchReader.from(<any> source); + const writer = await RecordBatchJSONWriter.writeAll(reader); + const buffer = await concatBuffersAsync(writer.toDOMStream()); + validateRecordBatchReader('json', 3, RecordBatchReader.from(toJSON(buffer))); + } + }); + + describe(`RecordBatchFileWriter`, () => { + + describe(`sync write/read`, () => { + + test(`Uint8Array`, file.buffer(validate)); + test(`Iterable`, file.iterable(validate)); + test('AsyncIterable', file.asyncIterable(validate)); + test('fs.FileHandle', file.fsFileHandle(validate)); + test('fs.ReadStream', file.fsReadableStream(validate)); + test('stream.Readable', file.nodeReadableStream(validate)); + test('whatwg.ReadableStream', file.whatwgReadableStream(validate)); + test('whatwg.ReadableByteStream', file.whatwgReadableByteStream(validate)); + test('Promise<AsyncIterable>', file.asyncIterable(wrapArgInPromise(validate))); + test('Promise<fs.FileHandle>', file.fsFileHandle(wrapArgInPromise(validate))); + test('Promise<fs.ReadStream>', file.fsReadableStream(wrapArgInPromise(validate))); + test('Promise<stream.Readable>', file.nodeReadableStream(wrapArgInPromise(validate))); + test('Promise<ReadableStream>', file.whatwgReadableStream(wrapArgInPromise(validate))); + test('Promise<ReadableByteStream>', file.whatwgReadableByteStream(wrapArgInPromise(validate))); + + async function validate(source: any) { + const reader = await RecordBatchReader.from(source); + const writer = await RecordBatchFileWriter.writeAll(reader); + const stream = await RecordBatchReader.from(writer.toDOMStream()); + await validateAsyncRecordBatchReader('file', 3, stream); + } + }); + + describe(`async write/read`, () => { + + test(`Uint8Array`, file.buffer(validate)); + test(`Iterable`, file.iterable(validate)); + test('AsyncIterable', file.asyncIterable(validate)); + test('fs.FileHandle', file.fsFileHandle(validate)); + test('fs.ReadStream', file.fsReadableStream(validate)); + test('stream.Readable', file.nodeReadableStream(validate)); + test('whatwg.ReadableStream', file.whatwgReadableStream(validate)); + test('whatwg.ReadableByteStream', file.whatwgReadableByteStream(validate)); + test('Promise<AsyncIterable>', file.asyncIterable(wrapArgInPromise(validate))); + test('Promise<fs.FileHandle>', file.fsFileHandle(wrapArgInPromise(validate))); + test('Promise<fs.ReadStream>', file.fsReadableStream(wrapArgInPromise(validate))); + test('Promise<stream.Readable>', file.nodeReadableStream(wrapArgInPromise(validate))); + test('Promise<ReadableStream>', file.whatwgReadableStream(wrapArgInPromise(validate))); + test('Promise<ReadableByteStream>', file.whatwgReadableByteStream(wrapArgInPromise(validate))); + + async function validate(source: any) { + const writer = new RecordBatchFileWriter(); + /* no await */ writer.writeAll(await RecordBatchReader.from(source)); + const reader = await RecordBatchReader.from(writer.toDOMStream()); + await validateAsyncRecordBatchReader('file', 3, reader); + } + }); + }); + + describe(`RecordBatchStreamWriter`, () => { + + describe(`sync write/read`, () => { + + test(`Uint8Array`, stream.buffer(validate)); + test(`Iterable`, stream.iterable(validate)); + test('AsyncIterable', stream.asyncIterable(validate)); + test('fs.FileHandle', stream.fsFileHandle(validate)); + test('fs.ReadStream', stream.fsReadableStream(validate)); + test('stream.Readable', stream.nodeReadableStream(validate)); + test('whatwg.ReadableStream', stream.whatwgReadableStream(validate)); + test('whatwg.ReadableByteStream', stream.whatwgReadableByteStream(validate)); + test('Promise<AsyncIterable>', stream.asyncIterable(wrapArgInPromise(validate))); + test('Promise<fs.FileHandle>', stream.fsFileHandle(wrapArgInPromise(validate))); + test('Promise<fs.ReadStream>', stream.fsReadableStream(wrapArgInPromise(validate))); + test('Promise<stream.Readable>', stream.nodeReadableStream(wrapArgInPromise(validate))); + test('Promise<ReadableStream>', stream.whatwgReadableStream(wrapArgInPromise(validate))); + test('Promise<ReadableByteStream>', stream.whatwgReadableByteStream(wrapArgInPromise(validate))); + + async function validate(source: any) { + const reader = await RecordBatchReader.from(source); + const writer = await RecordBatchStreamWriter.writeAll(reader); + const stream = await RecordBatchReader.from(writer.toDOMStream()); + await validateAsyncRecordBatchReader('stream', 3, stream); + } + }); + + describe(`async write/read`, () => { + + test(`Uint8Array`, stream.buffer(validate)); + test(`Iterable`, stream.iterable(validate)); + test('AsyncIterable', stream.asyncIterable(validate)); + test('fs.FileHandle', stream.fsFileHandle(validate)); + test('fs.ReadStream', stream.fsReadableStream(validate)); + test('stream.Readable', stream.nodeReadableStream(validate)); + test('whatwg.ReadableStream', stream.whatwgReadableStream(validate)); + test('whatwg.ReadableByteStream', stream.whatwgReadableByteStream(validate)); + test('Promise<AsyncIterable>', stream.asyncIterable(wrapArgInPromise(validate))); + test('Promise<fs.FileHandle>', stream.fsFileHandle(wrapArgInPromise(validate))); + test('Promise<fs.ReadStream>', stream.fsReadableStream(wrapArgInPromise(validate))); + test('Promise<stream.Readable>', stream.nodeReadableStream(wrapArgInPromise(validate))); + test('Promise<ReadableStream>', stream.whatwgReadableStream(wrapArgInPromise(validate))); + test('Promise<ReadableByteStream>', stream.whatwgReadableByteStream(wrapArgInPromise(validate))); + + async function validate(source: any) { + const writer = new RecordBatchStreamWriter(); + /* no await */ writer.writeAll(await RecordBatchReader.from(source)); + const reader = await RecordBatchReader.from(writer.toDOMStream()); + await validateAsyncRecordBatchReader('stream', 3, reader); + } + }); + }); + }); + } + + describe(`RecordBatchStreamWriter.throughDOM`, () => { + + const opts = { autoDestroy: false }; + const sleep = (n: number) => new Promise((r) => setTimeout(r, n)); + + it(`should write a stream of tables to the same output stream`, async () => { + + const tables = [] as Table[]; + const stream: ReadableStream<any> = from(generateRandomTables([10, 20, 30])) + // insert some asynchrony + .pipe(tap({ async next(table: Table) { tables.push(table); await sleep(1); } })) + .pipeThrough(RecordBatchStreamWriter.throughDOM(opts)); + + for await (const reader of RecordBatchReader.readAll(stream)) { + const sourceTable = tables.shift()!; + const streamTable = await Table.from(reader); + expect(streamTable).toEqualTable(sourceTable); + } + + expect(tables).toHaveLength(0); + expect(stream.locked).toBe(false); + }); + + it(`should write a stream of record batches to the same output stream`, async () => { + + const tables = [] as Table[]; + const stream = from(generateRandomTables([10, 20, 30])) + // insert some asynchrony + .pipe(tap({ async next(table: Table) { tables.push(table); await sleep(1); } })) + // flatMap from Table -> RecordBatches[] + .pipe(flatMap((table) => as(table.chunks))) + .pipeThrough(RecordBatchStreamWriter.throughDOM(opts)); + + for await (const reader of RecordBatchReader.readAll(stream)) { + const sourceTable = tables.shift()!; + const streamTable = await Table.from(reader); + expect(streamTable).toEqualTable(sourceTable); + } + + expect(tables).toHaveLength(0); + expect(stream.locked).toBe(false); + }); + }); + +})(); diff --git a/src/arrow/js/test/unit/ipc/writer/streams-node-tests.ts b/src/arrow/js/test/unit/ipc/writer/streams-node-tests.ts new file mode 100644 index 000000000..662129b1b --- /dev/null +++ b/src/arrow/js/test/unit/ipc/writer/streams-node-tests.ts @@ -0,0 +1,274 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { + generateRandomTables, + // generateDictionaryTables +} from '../../../data/tables'; + +import { from, as } from 'ix/asynciterable'; +import { tap, flatMap } from 'ix/asynciterable/operators'; +import 'ix/Ix.node'; + +import { + Table, + RecordBatchReader, + RecordBatchWriter, + RecordBatchFileWriter, + RecordBatchJSONWriter, + RecordBatchStreamWriter, +} from 'apache-arrow'; + +import { + ArrowIOTestHelper, + concatBuffersAsync +} from '../helpers'; + +import { + validateRecordBatchReader, + validateAsyncRecordBatchReader, + validateRecordBatchAsyncIterator +} from '../validate'; + +(() => { + + if (process.env.TEST_NODE_STREAMS !== 'true') { + return test('not testing node streams because process.env.TEST_NODE_STREAMS !== "true"', () => {}); + } + + for (const table of generateRandomTables([10, 20, 30])) { + + const file = ArrowIOTestHelper.file(table); + const json = ArrowIOTestHelper.json(table); + const stream = ArrowIOTestHelper.stream(table); + const name = `[\n ${table.schema.fields.join(',\n ')}\n]`; + + describe(`RecordBatchWriter.throughNode (${name})`, () => { + + describe('file', () => { + describe(`convert`, () => { + test('fs.ReadStream', file.fsReadableStream(validateConvert.bind(0, RecordBatchStreamWriter))); + test('stream.Readable', file.nodeReadableStream(validateConvert.bind(0, RecordBatchStreamWriter))); + }); + describe(`through`, () => { + test('fs.ReadStream', file.fsReadableStream(validateThrough.bind(0, RecordBatchFileWriter))); + test('stream.Readable', file.nodeReadableStream(validateThrough.bind(0, RecordBatchFileWriter))); + }); + }); + + describe('stream', () => { + describe(`convert`, () => { + test('fs.ReadStream', stream.fsReadableStream(validateConvert.bind(0, RecordBatchFileWriter))); + test('stream.Readable', stream.nodeReadableStream(validateConvert.bind(0, RecordBatchFileWriter))); + }); + describe(`through`, () => { + test('fs.ReadStream', stream.fsReadableStream(validateThrough.bind(0, RecordBatchStreamWriter))); + test('stream.Readable', stream.nodeReadableStream(validateThrough.bind(0, RecordBatchStreamWriter))); + }); + }); + + async function validateConvert(RBWImplementation: typeof RecordBatchWriter, source: NodeJS.ReadableStream) { + const stream = source + .pipe(RecordBatchReader.throughNode()) + .pipe(RBWImplementation.throughNode()); + const type = RBWImplementation === RecordBatchFileWriter ? 'file' : 'stream'; + await validateAsyncRecordBatchReader(type, 3, await RecordBatchReader.from(stream)); + } + + async function validateThrough(RBWImplementation: typeof RecordBatchWriter, source: NodeJS.ReadableStream) { + const stream = source + .pipe(RecordBatchReader.throughNode()) + .pipe(RBWImplementation.throughNode()) + .pipe(RecordBatchReader.throughNode()); + await validateRecordBatchAsyncIterator(3, stream[Symbol.asyncIterator]()); + } + }); + + describe(`toNodeStream (${name})`, () => { + + const wrapArgInPromise = (fn: (p: Promise<any>) => any) => (x: any) => fn(Promise.resolve(x)); + + describe(`RecordBatchJSONWriter`, () => { + + const toJSON = (x: any): { schema: any } => JSON.parse(`${Buffer.from(x)}`); + + test('Uint8Array', json.buffer((source) => validate(toJSON(source)))); + test('Promise<Uint8Array>', json.buffer((source) => validate(Promise.resolve(toJSON(source))))); + + async function validate(source: { schema: any } | Promise<{ schema: any }>) { + const reader = await RecordBatchReader.from(<any> source); + const writer = await RecordBatchJSONWriter.writeAll(reader); + const buffer = await concatBuffersAsync(writer.toNodeStream()); + validateRecordBatchReader('json', 3, RecordBatchReader.from(toJSON(buffer))); + } + }); + + describe(`RecordBatchFileWriter`, () => { + + describe(`sync write/read`, () => { + + test(`Uint8Array`, file.buffer(validate)); + test(`Iterable`, file.iterable(validate)); + test('AsyncIterable', file.asyncIterable(validate)); + test('fs.FileHandle', file.fsFileHandle(validate)); + test('fs.ReadStream', file.fsReadableStream(validate)); + test('stream.Readable', file.nodeReadableStream(validate)); + test('whatwg.ReadableStream', file.whatwgReadableStream(validate)); + test('whatwg.ReadableByteStream', file.whatwgReadableByteStream(validate)); + test('Promise<AsyncIterable>', file.asyncIterable(wrapArgInPromise(validate))); + test('Promise<fs.FileHandle>', file.fsFileHandle(wrapArgInPromise(validate))); + test('Promise<fs.ReadStream>', file.fsReadableStream(wrapArgInPromise(validate))); + test('Promise<stream.Readable>', file.nodeReadableStream(wrapArgInPromise(validate))); + test('Promise<ReadableStream>', file.whatwgReadableStream(wrapArgInPromise(validate))); + test('Promise<ReadableByteStream>', file.whatwgReadableByteStream(wrapArgInPromise(validate))); + + async function validate(source: any) { + const reader = await RecordBatchReader.from(source); + const writer = await RecordBatchFileWriter.writeAll(reader); + const stream = await RecordBatchReader.from(writer.toNodeStream()); + await validateAsyncRecordBatchReader('file', 3, stream); + } + }); + + describe(`async write/read`, () => { + + test(`Uint8Array`, file.buffer(validate)); + test(`Iterable`, file.iterable(validate)); + test('AsyncIterable', file.asyncIterable(validate)); + test('fs.FileHandle', file.fsFileHandle(validate)); + test('fs.ReadStream', file.fsReadableStream(validate)); + test('stream.Readable', file.nodeReadableStream(validate)); + test('whatwg.ReadableStream', file.whatwgReadableStream(validate)); + test('whatwg.ReadableByteStream', file.whatwgReadableByteStream(validate)); + test('Promise<AsyncIterable>', file.asyncIterable(wrapArgInPromise(validate))); + test('Promise<fs.FileHandle>', file.fsFileHandle(wrapArgInPromise(validate))); + test('Promise<fs.ReadStream>', file.fsReadableStream(wrapArgInPromise(validate))); + test('Promise<stream.Readable>', file.nodeReadableStream(wrapArgInPromise(validate))); + test('Promise<ReadableStream>', file.whatwgReadableStream(wrapArgInPromise(validate))); + test('Promise<ReadableByteStream>', file.whatwgReadableByteStream(wrapArgInPromise(validate))); + + async function validate(source: any) { + const writer = new RecordBatchFileWriter(); + /* no await */ writer.writeAll(await RecordBatchReader.from(source)); + const reader = await RecordBatchReader.from(writer.toNodeStream()); + await validateAsyncRecordBatchReader('file', 3, reader); + } + }); + }); + + describe(`RecordBatchStreamWriter`, () => { + + describe(`sync write/read`, () => { + + test(`Uint8Array`, stream.buffer(validate)); + test(`Iterable`, stream.iterable(validate)); + test('AsyncIterable', stream.asyncIterable(validate)); + test('fs.FileHandle', stream.fsFileHandle(validate)); + test('fs.ReadStream', stream.fsReadableStream(validate)); + test('stream.Readable', stream.nodeReadableStream(validate)); + test('whatwg.ReadableStream', stream.whatwgReadableStream(validate)); + test('whatwg.ReadableByteStream', stream.whatwgReadableByteStream(validate)); + test('Promise<AsyncIterable>', stream.asyncIterable(wrapArgInPromise(validate))); + test('Promise<fs.FileHandle>', stream.fsFileHandle(wrapArgInPromise(validate))); + test('Promise<fs.ReadStream>', stream.fsReadableStream(wrapArgInPromise(validate))); + test('Promise<stream.Readable>', stream.nodeReadableStream(wrapArgInPromise(validate))); + test('Promise<ReadableStream>', stream.whatwgReadableStream(wrapArgInPromise(validate))); + test('Promise<ReadableByteStream>', stream.whatwgReadableByteStream(wrapArgInPromise(validate))); + + async function validate(source: any) { + const reader = await RecordBatchReader.from(source); + const writer = await RecordBatchStreamWriter.writeAll(reader); + const stream = await RecordBatchReader.from(writer.toNodeStream()); + await validateAsyncRecordBatchReader('stream', 3, stream); + } + }); + + describe(`async write/read`, () => { + + test(`Uint8Array`, stream.buffer(validate)); + test(`Iterable`, stream.iterable(validate)); + test('AsyncIterable', stream.asyncIterable(validate)); + test('fs.FileHandle', stream.fsFileHandle(validate)); + test('fs.ReadStream', stream.fsReadableStream(validate)); + test('stream.Readable', stream.nodeReadableStream(validate)); + test('whatwg.ReadableStream', stream.whatwgReadableStream(validate)); + test('whatwg.ReadableByteStream', stream.whatwgReadableByteStream(validate)); + test('Promise<AsyncIterable>', stream.asyncIterable(wrapArgInPromise(validate))); + test('Promise<fs.FileHandle>', stream.fsFileHandle(wrapArgInPromise(validate))); + test('Promise<fs.ReadStream>', stream.fsReadableStream(wrapArgInPromise(validate))); + test('Promise<stream.Readable>', stream.nodeReadableStream(wrapArgInPromise(validate))); + test('Promise<ReadableStream>', stream.whatwgReadableStream(wrapArgInPromise(validate))); + test('Promise<ReadableByteStream>', stream.whatwgReadableByteStream(wrapArgInPromise(validate))); + + async function validate(source: any) { + const writer = new RecordBatchStreamWriter(); + /* no await */ writer.writeAll(await RecordBatchReader.from(source)); + const reader = await RecordBatchReader.from(writer.toNodeStream()); + await validateAsyncRecordBatchReader('stream', 3, reader); + } + }); + }); + }); + } + + describe(`RecordBatchStreamWriter.throughNode`, () => { + + const sleep = (n: number) => new Promise((r) => setTimeout(r, n)); + + it(`should write a stream of tables to the same output stream`, async () => { + + const tables = [] as Table[]; + const writer = RecordBatchStreamWriter.throughNode({ autoDestroy: false }); + const stream = from(generateRandomTables([10, 20, 30])) + // insert some asynchrony + .pipe(tap({ async next(table: Table) { tables.push(table); await sleep(1); } })) + .pipe(writer); + + for await (const reader of RecordBatchReader.readAll(stream)) { + const sourceTable = tables.shift()!; + const streamTable = await Table.from(reader); + expect(streamTable).toEqualTable(sourceTable); + } + + expect(tables).toHaveLength(0); + expect(writer.readable).toBe(false); + expect((writer as any).destroyed).toBe(true); + }); + + it(`should write a stream of record batches to the same output stream`, async () => { + + const tables = [] as Table[]; + const writer = RecordBatchStreamWriter.throughNode({ autoDestroy: false }); + const stream = from(generateRandomTables([10, 20, 30])) + // insert some asynchrony + .pipe(tap({ async next(table: Table) { tables.push(table); await sleep(1); } })) + .pipe(flatMap((table) => as(table.chunks))) + .pipe(writer); + + for await (const reader of RecordBatchReader.readAll(stream)) { + const sourceTable = tables.shift()!; + const streamTable = await Table.from(reader); + expect(streamTable).toEqualTable(sourceTable); + } + + expect(tables).toHaveLength(0); + expect(writer.readable).toBe(false); + expect((writer as any).destroyed).toBe(true); + }); + + }); +})(); diff --git a/src/arrow/js/test/unit/math-tests.ts b/src/arrow/js/test/unit/math-tests.ts new file mode 100644 index 000000000..7e3ffcd8f --- /dev/null +++ b/src/arrow/js/test/unit/math-tests.ts @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import * as Arrow from 'apache-arrow'; +const { float64ToUint16, uint16ToFloat64 } = Arrow.util; + +describe('Float16', () => { + test('Uint16 to Float64 works', () => { + + const uNaN = 0x7E00 /* NaN */; + const pInf = 0x7C00 /* 1/0 */; + const nInf = 0xFC00 /*-1/0 */; + let value = 0, expected = value; + + do { + + expected = value; + + // if exponent is all 1s, either Infinity or NaN + if ((value & 0x7C00) === 0x7C00) { + // if significand, must be NaN + if (((value << 6) & 0xFFFF) !== 0) { + expected = uNaN; + } else { + // otherwise +/- Infinity + expected = (value >>> 15) !== 0 ? nInf : pInf; + } + } + + expect(float64ToUint16(uint16ToFloat64(value))).toEqual(expected); + } while (++value < 65536); + }); +}); diff --git a/src/arrow/js/test/unit/recordbatch/record-batch-tests.ts b/src/arrow/js/test/unit/recordbatch/record-batch-tests.ts new file mode 100644 index 000000000..520c04f84 --- /dev/null +++ b/src/arrow/js/test/unit/recordbatch/record-batch-tests.ts @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import '../../jest-extensions'; +import { + Data, RecordBatch, + Vector, Int32Vector, Float32Vector, Float32, Int32, +} from 'apache-arrow'; +import { arange } from '../utils'; + +function numsRecordBatch(i32Len: number, f32Len: number) { + return RecordBatch.new({ + i32: Int32Vector.from(new Int32Array(arange(new Array(i32Len)))) as Int32Vector, + f32: Float32Vector.from(new Float32Array(arange(new Array(f32Len)))) as Float32Vector + }); +} + +describe(`RecordBatch`, () => { + describe(`new()`, () => { + + test(`creates a new RecordBatch from a Vector`, () => { + + const i32s = new Int32Array(arange(new Array<number>(10))); + + let i32 = Vector.new(Data.Int(new Int32(), 0, i32s.length, 0, null, i32s)); + expect(i32).toHaveLength(i32s.length); + expect(i32.nullCount).toBe(0); + + const batch = RecordBatch.new([i32], ['i32']); + i32 = batch.getChildAt(0) as Int32Vector; + + expect(batch.schema.fields[0].name).toBe('i32'); + expect(i32).toHaveLength(i32s.length); + expect(i32.nullCount).toBe(0); + + expect(i32).toEqualVector(Int32Vector.from(i32s)); + }); + + test(`creates a new RecordBatch from Vectors`, () => { + + const i32s = new Int32Array(arange(new Array<number>(10))); + const f32s = new Float32Array(arange(new Array<number>(10))); + + let i32 = Vector.new(Data.Int(new Int32(), 0, i32s.length, 0, null, i32s)); + let f32 = Vector.new(Data.Float(new Float32(), 0, f32s.length, 0, null, f32s)); + expect(i32).toHaveLength(i32s.length); + expect(f32).toHaveLength(f32s.length); + expect(i32.nullCount).toBe(0); + expect(f32.nullCount).toBe(0); + + const batch = RecordBatch.new([i32, f32], ['i32', 'f32']); + i32 = batch.getChildAt(0) as Int32Vector; + f32 = batch.getChildAt(1) as Float32Vector; + + expect(batch.schema.fields[0].name).toBe('i32'); + expect(batch.schema.fields[1].name).toBe('f32'); + expect(i32).toHaveLength(i32s.length); + expect(f32).toHaveLength(f32s.length); + expect(i32.nullCount).toBe(0); + expect(f32.nullCount).toBe(0); + + expect(i32).toEqualVector(Int32Vector.from(i32s)); + expect(f32).toEqualVector(Float32Vector.from(f32s)); + }); + + test(`creates a new RecordBatch from Vectors with different lengths`, () => { + + const i32s = new Int32Array(arange(new Array<number>(20))); + const f32s = new Float32Array(arange(new Array<number>(8))); + + let i32 = Int32Vector.from(i32s); + let f32 = Float32Vector.from(f32s); + + expect(i32).toHaveLength(i32s.length); + expect(f32).toHaveLength(f32s.length); + expect(i32.nullCount).toBe(0); + expect(f32.nullCount).toBe(0); + + const batch = RecordBatch.new([i32, f32]); + i32 = batch.getChildAt(0) as Int32Vector; + f32 = batch.getChildAt(1) as Float32Vector; + + expect(batch.schema.fields[0].name).toBe('0'); + expect(batch.schema.fields[1].name).toBe('1'); + expect(i32).toHaveLength(i32s.length); + expect(f32).toHaveLength(i32s.length); // new length should be the same as the longest sibling + expect(i32.nullCount).toBe(0); + expect(f32.nullCount).toBe(i32s.length - f32s.length); + + const f32Expected = Data.Float( + f32.type, 0, i32s.length, + i32s.length - f32s.length, + new Uint8Array(8).fill(255, 0, 1), f32s); + + expect(i32).toEqualVector(Int32Vector.from(i32s)); + expect(f32).toEqualVector(new Float32Vector(f32Expected)); + }); + }); + + describe(`select()`, () => { + test(`can select recordbatch children by name`, () => { + const batch = numsRecordBatch(32, 27); + const i32sBatch = batch.select('i32'); + expect(i32sBatch.numCols).toBe(1); + expect(i32sBatch).toHaveLength(32); + }); + }); + describe(`selectAt()`, () => { + test(`can select recordbatch children by index`, () => { + const batch = numsRecordBatch(32, 45); + const f32sBatch = batch.selectAt(1); + expect(f32sBatch.numCols).toBe(1); + expect(f32sBatch).toHaveLength(45); + }); + }); +}); diff --git a/src/arrow/js/test/unit/table-tests.ts b/src/arrow/js/test/unit/table-tests.ts new file mode 100644 index 000000000..2f138182b --- /dev/null +++ b/src/arrow/js/test/unit/table-tests.ts @@ -0,0 +1,406 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import '../jest-extensions'; +import { + Data, Schema, Field, Table, RecordBatch, Column, + Vector, Int32Vector, Float32Vector, Utf8Vector, DictionaryVector, + Struct, Float32, Int32, Dictionary, Utf8, Int8 +} from 'apache-arrow'; +import { arange } from './utils'; + +const NAMES = ['f32', 'i32', 'dictionary'] as (keyof TestDataSchema)[]; +const F32 = 0, I32 = 1, DICT = 2; +export const test_data = [ + { + name: `single record batch`, + table: getSingleRecordBatchTable, + // Use Math.fround to coerce to float32 + values: () => [ + [Math.fround(-0.3), -1, 'a'], + [Math.fround(-0.2), 1, 'b'], + [Math.fround(-0.1), -1, 'c'], + [Math.fround(0), 1, 'a'], + [Math.fround(0.1), -1, 'b'], + [Math.fround(0.2), 1, 'c'], + [Math.fround(0.3), -1, 'a'] + ] + }, { + name: `multiple record batches`, + table: getMultipleRecordBatchesTable, + values: () => [ + [Math.fround(-0.3), -1, 'a'], + [Math.fround(-0.2), 1, 'b'], + [Math.fround(-0.1), -1, 'c'], + [Math.fround(0), 1, 'a'], + [Math.fround(0.1), -1, 'b'], + [Math.fround(0.2), 1, 'c'], + [Math.fround(0.3), -1, 'a'], + [Math.fround(0.2), 1, 'b'], + [Math.fround(0.1), -1, 'c'], + ] + }, { + name: `struct`, + table: () => Table.fromStruct(getStructTable().getColumn('struct')!), + // Use Math.fround to coerce to float32 + values: () => [ + [Math.fround(-0.3), -1, 'a'], + [Math.fround(-0.2), 1, 'b'], + [Math.fround(-0.1), -1, 'c'], + [Math.fround(0), 1, 'a'], + [Math.fround(0.1), -1, 'b'], + [Math.fround(0.2), 1, 'c'], + [Math.fround(0.3), -1, 'a'] + ] + }, +]; + +function compareBatchAndTable(source: Table, offset: number, batch: RecordBatch, table: Table) { + expect(batch).toHaveLength(table.length); + expect(table.numCols).toEqual(source.numCols); + expect(batch.numCols).toEqual(source.numCols); + for (let i = -1, n = source.numCols; ++i < n;) { + const v0 = source.getColumnAt(i)!.slice(offset, offset + batch.length); + const v1 = batch.getChildAt(i); + const v2 = table.getColumnAt(i); + const name = source.schema.fields[i].name; + expect([v1, `batch`, name]).toEqualVector([v0, `source`]); + expect([v2, `table`, name]).toEqualVector([v0, `source`]); + } +} + +describe(`Table`, () => { + test(`can create an empty table`, () => { + expect(Table.empty()).toHaveLength(0); + }); + test(`Table.from([]) creates an empty table`, () => { + expect(Table.from([])).toHaveLength(0); + }); + test(`Table.from() creates an empty table`, () => { + expect(Table.from()).toHaveLength(0); + }); + + describe(`new()`, () => { + test(`creates an empty Table with Columns`, () => { + let i32 = Column.new('i32', Data.new(new Int32(), 0, 0)); + let f32 = Column.new('f32', Data.new(new Float32(), 0, 0)); + const table = Table.new(i32, f32); + i32 = table.getColumn('i32')!; + f32 = table.getColumn('f32')!; + expect(table).toHaveLength(0); + expect(i32).toHaveLength(0); + expect(f32).toHaveLength(0); + expect(i32.toArray()).toBeInstanceOf(Int32Array); + expect(f32.toArray()).toBeInstanceOf(Float32Array); + }); + + test(`creates a new Table from a Column`, () => { + + const i32s = new Int32Array(arange(new Array<number>(10))); + + let i32 = Column.new('i32', Data.Int(new Int32(), 0, i32s.length, 0, null, i32s)); + expect(i32.name).toBe('i32'); + expect(i32).toHaveLength(i32s.length); + expect(i32.nullable).toBe(true); + expect(i32.nullCount).toBe(0); + + const table = Table.new(i32); + i32 = table.getColumnAt(0)!; + + expect(i32.name).toBe('i32'); + expect(i32).toHaveLength(i32s.length); + expect(i32.nullable).toBe(true); + expect(i32.nullCount).toBe(0); + + expect(i32).toEqualVector(Int32Vector.from(i32s)); + }); + + test(`creates a new Table from Columns`, () => { + + const i32s = new Int32Array(arange(new Array<number>(10))); + const f32s = new Float32Array(arange(new Array<number>(10))); + + let i32 = Column.new('i32', Data.Int(new Int32(), 0, i32s.length, 0, null, i32s)); + let f32 = Column.new('f32', Data.Float(new Float32(), 0, f32s.length, 0, null, f32s)); + expect(i32.name).toBe('i32'); + expect(f32.name).toBe('f32'); + expect(i32).toHaveLength(i32s.length); + expect(f32).toHaveLength(f32s.length); + expect(i32.nullable).toBe(true); + expect(f32.nullable).toBe(true); + expect(i32.nullCount).toBe(0); + expect(f32.nullCount).toBe(0); + + const table = Table.new(i32, f32); + i32 = table.getColumnAt(0)!; + f32 = table.getColumnAt(1)!; + + expect(i32.name).toBe('i32'); + expect(f32.name).toBe('f32'); + expect(i32).toHaveLength(i32s.length); + expect(f32).toHaveLength(f32s.length); + expect(i32.nullable).toBe(true); + expect(f32.nullable).toBe(true); + expect(i32.nullCount).toBe(0); + expect(f32.nullCount).toBe(0); + + expect(i32).toEqualVector(Int32Vector.from(i32s)); + expect(f32).toEqualVector(Float32Vector.from(f32s)); + }); + + test(`creates a new Table from Columns with different lengths`, () => { + + const i32s = new Int32Array(arange(new Array<number>(20))); + const f32s = new Float32Array(arange(new Array<number>(8))); + + let i32 = Column.new('i32', Int32Vector.from(i32s)); + let f32 = Column.new('f32', Float32Vector.from(f32s)); + + expect(i32.name).toBe('i32'); + expect(f32.name).toBe('f32'); + expect(i32).toHaveLength(i32s.length); + expect(f32).toHaveLength(f32s.length); + expect(i32.nullable).toBe(true); + expect(f32.nullable).toBe(true); + expect(i32.nullCount).toBe(0); + expect(f32.nullCount).toBe(0); + + const table = Table.new([i32, f32]); + i32 = table.getColumnAt(0)!; + f32 = table.getColumnAt(1)!; + + expect(i32.name).toBe('i32'); + expect(f32.name).toBe('f32'); + expect(i32).toHaveLength(i32s.length); + expect(f32).toHaveLength(i32s.length); // new length should be the same as the longest sibling + expect(i32.nullable).toBe(true); + expect(f32.nullable).toBe(true); // true, with 12 additional nulls + expect(i32.nullCount).toBe(0); + expect(f32.nullCount).toBe(i32s.length - f32s.length); + + const f32Expected = Data.Float( + f32.type, 0, i32s.length, + i32s.length - f32s.length, + new Uint8Array(8).fill(255, 0, 1), f32s); + + expect(i32).toEqualVector(Int32Vector.from(i32s)); + expect(f32).toEqualVector(new Float32Vector(f32Expected)); + }); + + test(`creates a new Table from Columns with different lengths and number of inner chunks`, () => { + + const i32s = new Int32Array(arange(new Array<number>(20))); + const f32s = new Float32Array(arange(new Array<number>(16))); + + let i32 = Column.new('i32', Int32Vector.from(i32s)); + let f32 = Column.new('f32', Float32Vector.from(f32s.slice(0, 8)), Float32Vector.from(f32s.slice(8, 16))); + + expect(i32.name).toBe('i32'); + expect(f32.name).toBe('f32'); + expect(i32).toHaveLength(i32s.length); + expect(f32).toHaveLength(f32s.length); + expect(i32.nullable).toBe(true); + expect(f32.nullable).toBe(true); + expect(i32.nullCount).toBe(0); + expect(f32.nullCount).toBe(0); + + const table = Table.new({ i32Renamed: i32, f32Renamed: f32 }); + i32 = table.getColumn('i32Renamed'); + f32 = table.getColumn('f32Renamed'); + + expect(i32.name).toBe('i32Renamed'); + expect(f32.name).toBe('f32Renamed'); + expect(i32).toHaveLength(i32s.length); + expect(f32).toHaveLength(i32s.length); // new length should be the same as the longest sibling + expect(i32.nullable).toBe(true); + expect(f32.nullable).toBe(true); // true, with 4 additional nulls + expect(i32.nullCount).toBe(0); + expect(f32.nullCount).toBe(i32s.length - f32s.length); + + const f32Expected = Data.Float( + f32.type, 0, i32s.length, + i32s.length - f32s.length, + new Uint8Array(8).fill(255, 0, 2), f32s); + + expect(i32).toEqualVector(Int32Vector.from(i32s)); + expect(f32).toEqualVector(new Float32Vector(f32Expected)); + }); + + test(`creates a new Table from Typed Arrays`, () => { + let i32s = Int32Array.from({length: 10}, (_, i) => i); + let f32s = Float32Array.from({length: 10}, (_, i) => i); + const table = Table.new({ i32s, f32s }); + const i32 = table.getColumn('i32s')!; + const f32 = table.getColumn('f32s')!; + + expect(table).toHaveLength(10); + expect(i32).toHaveLength(10); + expect(f32).toHaveLength(10); + expect(i32.toArray()).toBeInstanceOf(Int32Array); + expect(f32.toArray()).toBeInstanceOf(Float32Array); + expect(i32.toArray()).toEqual(i32s); + expect(f32.toArray()).toEqual(f32s); + }); + }); + + test(`Table.serialize() serializes sliced RecordBatches`, () => { + + const table = getSingleRecordBatchTable(); + const batch = table.chunks[0], half = batch.length / 2 | 0; + + // First compare what happens when slicing from the batch level + let [batch1, batch2] = [batch.slice(0, half), batch.slice(half)]; + + compareBatchAndTable(table, 0, batch1, Table.from(new Table(batch1).serialize())); + compareBatchAndTable(table, half, batch2, Table.from(new Table(batch2).serialize())); + + // Then compare what happens when creating a RecordBatch by slicing each child individually + batch1 = new RecordBatch(batch1.schema, batch1.length, batch1.schema.fields.map((_, i) => { + return batch.getChildAt(i)!.slice(0, half); + })); + + batch2 = new RecordBatch(batch2.schema, batch2.length, batch2.schema.fields.map((_, i) => { + return batch.getChildAt(i)!.slice(half); + })); + + compareBatchAndTable(table, 0, batch1, Table.from(new Table(batch1).serialize())); + compareBatchAndTable(table, half, batch2, Table.from(new Table(batch2).serialize())); + }); + + for (let datum of test_data) { + describe(datum.name, () => { + test(`has the correct length`, () => { + const table = datum.table(); + const values = datum.values(); + expect(table).toHaveLength(values.length); + }); + test(`gets expected values`, () => { + const table = datum.table(); + const values = datum.values(); + for (let i = -1; ++i < values.length;) { + const row = table.get(i); + const expected = values[i]; + expect(row.f32).toEqual(expected[F32]); + expect(row.i32).toEqual(expected[I32]); + expect(row.dictionary).toEqual(expected[DICT]); + } + }); + test(`iterates expected values`, () => { + let i = 0; + const table = datum.table(); + const values = datum.values(); + for (let row of table) { + const expected = values[i++]; + expect(row.f32).toEqual(expected[F32]); + expect(row.i32).toEqual(expected[I32]); + expect(row.dictionary).toEqual(expected[DICT]); + } + }); + test(`serialize and de-serialize is a no-op`, () => { + const table = datum.table(); + const clone = Table.from(table.serialize()); + expect(clone).toEqualTable(table); + }); + + test(`count() returns the correct length`, () => { + const table = datum.table(); + const values = datum.values(); + expect(table.count()).toEqual(values.length); + }); + test(`getColumnIndex`, () => { + const table = datum.table(); + expect(table.getColumnIndex('i32')).toEqual(I32); + expect(table.getColumnIndex('f32')).toEqual(F32); + expect(table.getColumnIndex('dictionary')).toEqual(DICT); + }); + + const table = datum.table(); + const values = datum.values(); + + test(`table.select() basic tests`, () => { + let selected = table.select('f32', 'dictionary'); + expect(selected.schema.fields).toHaveLength(2); + expect(selected.schema.fields[0]).toEqual(table.schema.fields[0]); + expect(selected.schema.fields[1]).toEqual(table.schema.fields[2]); + + expect(selected).toHaveLength(values.length); + let idx = 0, expected_row; + for (let row of selected) { + expected_row = values[idx++]; + expect(row.f32).toEqual(expected_row[F32]); + expect(row.dictionary).toEqual(expected_row[DICT]); + } + }); + }); + } +}); + +type TestDataSchema = { f32: Float32; i32: Int32; dictionary: Dictionary<Utf8, Int8> }; + +function getTestVectors(f32Values: number[], i32Values: number[], dictIndices: number[]) { + + const values = Utf8Vector.from(['a', 'b', 'c']); + const i32Data = Data.Int(new Int32(), 0, i32Values.length, 0, null, i32Values); + const f32Data = Data.Float(new Float32(), 0, f32Values.length, 0, null, f32Values); + + return [Vector.new(f32Data), Vector.new(i32Data), DictionaryVector.from(values, new Int8(), dictIndices)]; +} + +function getSingleRecordBatchTable() { + const vectors = getTestVectors( + [-0.3, -0.2, -0.1, 0, 0.1, 0.2, 0.3], + [-1, 1, -1, 1, -1, 1, -1], + [0, 1, 2, 0, 1, 2, 0] + ); + + return Table.new<TestDataSchema>(vectors, NAMES); +} + +function getMultipleRecordBatchesTable() { + + const types = getTestVectors([], [], []).map((vec) => vec.type); + const fields = NAMES.map((name, i) => Field.new(name, types[i])); + const schema = new Schema<TestDataSchema>(fields); + + const b1 = new RecordBatch(schema, 3, getTestVectors( + [-0.3, -0.2, -0.1], + [-1, 1, -1], + [0, 1, 2] + )); + + const b2 = new RecordBatch(schema, 3, getTestVectors( + [0, 0.1, 0.2], + [1, -1, 1], + [0, 1, 2] + )); + + const b3 = new RecordBatch(schema, 3, getTestVectors( + [0.3, 0.2, 0.1], + [-1, 1, -1], + [0, 1, 2] + )); + + return new Table<TestDataSchema>([b1, b2, b3]); +} + +function getStructTable() { + const table = getSingleRecordBatchTable(); + const struct = new Struct<TestDataSchema>(table.schema.fields); + const children = table.schema.fields.map((_, i) => table.getColumnAt(i)!); + const structVec = Vector.new(Data.Struct(struct, 0, table.length, 0, null, children)); + return Table.new<{ struct: Struct<TestDataSchema> }>([structVec], ['struct']); +} diff --git a/src/arrow/js/test/unit/table/assign-tests.ts b/src/arrow/js/test/unit/table/assign-tests.ts new file mode 100644 index 000000000..fa1dacbc6 --- /dev/null +++ b/src/arrow/js/test/unit/table/assign-tests.ts @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* eslint-disable jest/no-standalone-expect */ + +import '../../jest-extensions'; +import { zip } from 'ix/iterable'; +import * as generate from '../../generate-test-data'; +import { validateTable } from '../generated-data-validators'; +import { + Schema, Field, DataType, Int32, Float32, Utf8 +} from 'apache-arrow'; + +const toSchema = (...xs: [string, DataType][]) => new Schema(xs.map((x) => new Field(...x))); +const schema1 = toSchema(['a', new Int32()], ['b', new Float32()], ['c', new Utf8()]); +const partialOverlapWith1 = toSchema(['a', new Int32()], ['b', new Float32()], ['f', new Utf8()]); +const schema2 = toSchema(['d', new Int32()], ['e', new Float32()], ['f', new Utf8()]); + +describe('Table.assign()', () => { + describe(`should assign non-overlapping fields`, () => { + const lhs = generate.table([20], schema1); + const rhs = generate.table([20], schema2); + const table = lhs.table.assign(rhs.table); + const f = assignGeneratedTables(lhs, rhs); + expect(table.schema.fields.map((f) => f.name)).toEqual(['a', 'b', 'c', 'd', 'e', 'f']); + validateTable({ ...f([0,1,2], [3,4,5]), table }).run(); + }); + describe(`should assign partially-overlapping fields`, () => { + const lhs = generate.table([20], schema1); + const rhs = generate.table([20], partialOverlapWith1); + const table = lhs.table.assign(rhs.table); + const f = assignGeneratedTables(lhs, rhs); + expect(table.schema.fields.map((f) => f.name)).toEqual(['a', 'b', 'c', 'f']); + // eslint-disable-next-line no-sparse-arrays + validateTable({ ...f([ , , 2], [0,1,3]), table }).run(); + }); + describe(`should assign completely-overlapping fields`, () => { + const lhs = generate.table([20], schema2); + const rhs = generate.table([20], schema2); + const table = lhs.table.assign(rhs.table); + const f = assignGeneratedTables(lhs, rhs); + expect(table.schema.fields.map((f) => f.name)).toEqual(['d', 'e', 'f']); + // eslint-disable-next-line no-sparse-arrays + validateTable({ ...f([ , , ], [0,1,2]), table }).run(); + }); +}); + +function assignGeneratedTables(lhs: generate.GeneratedTable, rhs: generate.GeneratedTable) { + return function createAssignedTestData(lhsIndices: any[], rhsIndices: any[]) { + const pluckLhs = (xs: any[], ys: any[] = []) => lhsIndices.reduce((ys, i, j) => { + if (i !== undefined) { ys[i] = xs ? xs[j] : null; } + return ys; + }, ys); + const pluckRhs = (xs: any[], ys: any[] = []) => rhsIndices.reduce((ys, i, j) => { + if (i !== undefined) { ys[i] = xs ? xs[j] : null; } + return ys; + }, ys); + const cols = () => [...pluckLhs(lhs.cols(), pluckRhs(rhs.cols()))]; + const keys = () => [...pluckLhs(lhs.keys(), pluckRhs(rhs.keys()))]; + const rows = () => [...zip(lhs.rows(), rhs.rows())].map(([x, y]) => [...pluckLhs(x, pluckRhs(y))]); + const colBatches = [...zip(lhs.colBatches, rhs.colBatches)].map(([x, y]) => () => [...pluckLhs(x(), pluckRhs(y()))]); + const keyBatches = [...zip(lhs.keyBatches, rhs.keyBatches)].map(([x, y]) => () => [...pluckLhs(x(), pluckRhs(y()))]); + const rowBatches = [...zip(lhs.rowBatches, rhs.rowBatches)].map(([x, y]) => () => [...zip(x(), y())].map(([x, y]) => [...pluckLhs(x, pluckRhs(y))])); + return { cols, keys, rows, colBatches, keyBatches, rowBatches }; + }; +} diff --git a/src/arrow/js/test/unit/table/serialize-tests.ts b/src/arrow/js/test/unit/table/serialize-tests.ts new file mode 100644 index 000000000..5eb211763 --- /dev/null +++ b/src/arrow/js/test/unit/table/serialize-tests.ts @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import '../../jest-extensions'; +import * as generate from '../../generate-test-data'; +import { + Table, Schema, Field, DataType, Dictionary, Int32, Float32, Utf8, Null, Int32Vector +} from 'apache-arrow'; + +const toSchema = (...xs: [string, DataType][]) => new Schema(xs.map((x) => new Field(...x))); +const schema1 = toSchema(['a', new Int32()], ['b', new Float32()], ['c', new Dictionary(new Utf8(), new Int32())]); +const schema2 = toSchema(['d', new Int32()], ['e', new Float32()], ['f', new Utf8()]); +const nullSchema = new Schema([new Field('null', new Null())]); + +schema1.metadata.set('foo', 'bar'); + +function createTable<T extends { [key: string]: DataType } = any>(schema: Schema<T>, chunkLengths: number[]) { + return generate.table(chunkLengths, schema).table; +} + +describe('Table#serialize()', () => { + + test(`doesn't swap the order of buffers that share the same underlying ArrayBuffer but are in a different order`, () => { + const values = new Int32Array([0, 1, 2, 3, 4, 5, 6, 7]); + const expected = values.slice(); + const x = Int32Vector.from(values.subarray(4, 8)); // back + const y = Int32Vector.from(values.subarray(0, 4)); // front + const source = Table.new([x, y], ['x', 'y']); + const table = Table.from(source.serialize()); + expect(table.getColumn('x').toArray()).toEqual(expected.subarray(4, 8)); + expect(table.getColumn('y').toArray()).toEqual(expected.subarray(0, 4)); + }); + + test(`Table#empty round-trips through serialization`, () => { + const source = Table.empty(); + source.schema.metadata.set('foo', 'bar'); + expect(source).toHaveLength(0); + expect(source.numCols).toBe(0); + const result = Table.from(source.serialize()); + expect(result).toEqualTable(source); + expect(result.schema.metadata.get('foo')).toEqual('bar'); + }); + + test(`Schema metadata round-trips through serialization`, () => { + const source = createTable(schema1, [20]); + expect(source).toHaveLength(20); + expect(source.numCols).toBe(3); + const result = Table.from(source.serialize()); + expect(result).toEqualTable(source); + expect(result.schema.metadata.get('foo')).toEqual('bar'); + }); + + test(`Table#assign an empty Table to a Table with a zero-length Null column round-trips through serialization`, () => { + const table1 = new Table(nullSchema); + const table2 = Table.empty(); + const source = table1.assign(table2); + expect(source).toHaveLength(0); + expect(source.numCols).toBe(1); + const result = Table.from(source.serialize()); + expect(result).toEqualTable(source); + }); + + const chunkLengths = [] as number[]; + for (let i = -1; ++i < 3;) { + chunkLengths[i * 2] = (Math.random() * 100) | 0; + chunkLengths[i * 2 + 1] = 0; + const table = <T extends { [key: string]: DataType } = any>(schema: Schema<T>) => createTable(schema, chunkLengths); + test(`Table#select round-trips through serialization`, () => { + const source = table(schema1).select('a', 'c'); + expect(source.numCols).toBe(2); + const result = Table.from(source.serialize()); + expect(result).toEqualTable(source); + }); + test(`Table#selectAt round-trips through serialization`, () => { + const source = table(schema1).selectAt(0, 2); + expect(source.numCols).toBe(2); + const result = Table.from(source.serialize()); + expect(result).toEqualTable(source); + }); + test(`Table#assign round-trips through serialization`, () => { + const source = table(schema1).assign(table(schema2)); + expect(source.numCols).toBe(6); + const result = Table.from(source.serialize()); + expect(result).toEqualTable(source); + expect(result.schema.metadata.get('foo')).toEqual('bar'); + }); + test(`Table#assign with an empty table round-trips through serialization`, () => { + const table1 = table(schema1); + const source = table1.assign(Table.empty()); + expect(source.numCols).toBe(table1.numCols); + expect(source).toHaveLength(table1.length); + const result = Table.from(source.serialize()); + expect(result).toEqualTable(source); + expect(result.schema.metadata.get('foo')).toEqual('bar'); + }); + test(`Table#assign with a zero-length Null column round-trips through serialization`, () => { + const table1 = new Table(nullSchema); + const table2 = table(schema1); + const source = table1.assign(table2); + expect(source).toHaveLength(table2.length); + expect(source.numCols).toBe(4); + const result = Table.from(source.serialize()); + expect(result).toEqualTable(source); + expect(result.schema.metadata.get('foo')).toEqual('bar'); + }); + test(`Table#assign with different lengths and number of chunks round-trips through serialization`, () => { + const table1 = table(schema1); + const table2 = createTable(schema2, [102, 4, 10, 97, 10, 2, 4]); + const source = table1.assign(table2); + expect(source.numCols).toBe(6); + expect(source).toHaveLength(Math.max(table1.length, table2.length)); + const result = Table.from(source.serialize()); + expect(result).toEqualTable(source); + expect(result.schema.metadata.get('foo')).toEqual('bar'); + }); + test(`Table#select with Table#assign the result of Table#selectAt round-trips through serialization`, () => { + const table1 = table(schema1); + const table2 = table(schema2); + const source = table1.select('a', 'c').assign(table2.selectAt(2)); + expect(source.numCols).toBe(3); + const result = Table.from(source.serialize()); + expect(result).toEqualTable(source); + expect(result.schema.metadata.get('foo')).toEqual('bar'); + }); + test(`Table#slice round-trips through serialization`, () => { + const table1 = table(schema1); + const length = table1.length; + const [begin, end] = [length * .25, length * .75].map((x) => x | 0); + const source = table1.slice(begin, end); + expect(source.numCols).toBe(3); + expect(source).toHaveLength(end - begin); + const result = Table.from(source.serialize()); + expect(result).toEqualTable(source); + expect(result.schema.metadata.get('foo')).toEqual('bar'); + }); + test(`Table#concat of two slices round-trips through serialization`, () => { + const table1 = table(schema1); + const length = table1.length; + const [begin1, end1] = [length * .10, length * .20].map((x) => x | 0); + const [begin2, end2] = [length * .80, length * .90].map((x) => x | 0); + const slice1 = table1.slice(begin1, end1); + const slice2 = table1.slice(begin2, end2); + const source = slice1.concat(slice2); + expect(slice1).toHaveLength(end1 - begin1); + expect(slice2).toHaveLength(end2 - begin2); + expect(source).toHaveLength((end1 - begin1) + (end2 - begin2)); + [slice1, slice2, source].forEach((x) => expect(x.numCols).toBe(3)); + const result = Table.from(source.serialize()); + expect(result).toEqualTable(source); + expect(result.schema.metadata.get('foo')).toEqual('bar'); + }); + } +}); diff --git a/src/arrow/js/test/unit/utils-tests.ts b/src/arrow/js/test/unit/utils-tests.ts new file mode 100644 index 000000000..985bec7aa --- /dev/null +++ b/src/arrow/js/test/unit/utils-tests.ts @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { isTypedArray } from 'apache-arrow'; + +describe('isTypedArray', () => { + test('works for typed arrays', () => { + expect(isTypedArray(new Int8Array())).toBeTruthy(); + expect(isTypedArray(new Int32Array())).toBeTruthy(); + expect(isTypedArray(new BigInt64Array())).toBeTruthy(); + }); + + test('does not recognize arrays, buffers, or data views', () => { + expect(isTypedArray(new Array([1, 2, 3]))).toBeFalsy(); + expect(isTypedArray(new ArrayBuffer(10))).toBeFalsy(); + expect(isTypedArray(new DataView(new ArrayBuffer(10)))).toBeFalsy(); + }); +}); diff --git a/src/arrow/js/test/unit/utils.ts b/src/arrow/js/test/unit/utils.ts new file mode 100644 index 000000000..c57de487f --- /dev/null +++ b/src/arrow/js/test/unit/utils.ts @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +export function arange<T extends { length: number; [n: number]: number }>(arr: T, n = arr.length) { + for (let i = -1; ++i < n; arr[i] = i) { } + return arr; +} diff --git a/src/arrow/js/test/unit/vector/bool-vector-tests.ts b/src/arrow/js/test/unit/vector/bool-vector-tests.ts new file mode 100644 index 000000000..41c53da60 --- /dev/null +++ b/src/arrow/js/test/unit/vector/bool-vector-tests.ts @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Data, Bool, Vector, BoolVector } from 'apache-arrow'; + +const newBoolVector = (length: number, data: Uint8Array) => Vector.new(Data.Bool(new Bool(), 0, length, 0, null, data)); + +describe(`BoolVector`, () => { + const values = [true, true, false, true, true, false, false, false]; + const n = values.length; + const vector = newBoolVector(n, new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0])); + test(`gets expected values`, () => { + let i = -1; + while (++i < n) { + expect(vector.get(i)).toEqual(values[i]); + } + }); + test(`iterates expected values`, () => { + let i = -1; + for (let v of vector) { + expect(++i).toBeLessThan(n); + expect(v).toEqual(values[i]); + } + }); + test(`indexOf returns expected values`, () => { + for (let test_value of [true, false]) { + const expected = values.indexOf(test_value); + expect(vector.indexOf(test_value)).toEqual(expected); + } + }); + test(`indexOf returns -1 when value not found`, () => { + const v = newBoolVector(3, new Uint8Array([0xFF])); + expect(v.indexOf(false)).toEqual(-1); + }); + test(`can set values to true and false`, () => { + const v = newBoolVector(n, new Uint8Array([27, 0, 0, 0, 0, 0, 0, 0])); + const expected1 = [true, true, false, true, true, false, false, false]; + const expected2 = [true, true, true, true, true, false, false, false]; + const expected3 = [true, true, false, false, false, false, true, true]; + function validate(expected: boolean[]) { + for (let i = -1; ++i < n;) { + expect(v.get(i)).toEqual(expected[i]); + } + } + validate(expected1); + v.set(2, true); + validate(expected2); + v.set(2, false); + validate(expected1); + v.set(3, false); + v.set(4, false); + v.set(6, true); + v.set(7, true); + validate(expected3); + v.set(3, true); + v.set(4, true); + v.set(6, false); + v.set(7, false); + validate(expected1); + }); + test(`packs 0 values`, () => { + const expected = new Uint8Array(64); + expect(BoolVector.from([]).values).toEqual(expected); + }); + test(`packs 3 values`, () => { + const expected = new Uint8Array(64); + expected[0] = 5; + expect(BoolVector.from([ + true, false, true + ]).values).toEqual(expected); + }); + test(`packs 8 values`, () => { + const expected = new Uint8Array(64); + expected[0] = 27; + expect(BoolVector.from([ + true, true, false, true, true, false, false, false + ]).values).toEqual(expected); + }); + test(`packs 25 values`, () => { + const expected = new Uint8Array(64); + expected[0] = 27; + expected[1] = 216; + expect(BoolVector.from([ + true, true, false, true, true, false, false, false, + false, false, false, true, true, false, true, true, + false + ]).values).toEqual(expected); + }); + test(`from with boolean Array packs values`, () => { + const expected = new Uint8Array(64); + expected[0] = 5; + expect(BoolVector + .from([true, false, true]) + .slice().values + ).toEqual(expected); + }); +}); diff --git a/src/arrow/js/test/unit/vector/date-vector-tests.ts b/src/arrow/js/test/unit/vector/date-vector-tests.ts new file mode 100644 index 000000000..4658633ba --- /dev/null +++ b/src/arrow/js/test/unit/vector/date-vector-tests.ts @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Table, DateDay, DateMillisecond } from 'apache-arrow'; + +describe(`DateVector`, () => { + it('returns days since the epoch as correct JS Dates', () => { + const table = Table.from(test_data); + const expectedMillis = expectedMillis32(); + const date32 = table.getColumnAt<DateDay>(0)!; + for (const date of date32) { + const millis = expectedMillis.shift(); + expect(date).toEqual(millis === null ? null : new Date(millis!)); + } + }); + it('returns millisecond longs since the epoch as correct JS Dates', () => { + const table = Table.from(test_data); + const expectedMillis = expectedMillis64(); + const date64 = table.getColumnAt<DateMillisecond>(1)!; + for (const date of date64) { + const millis = expectedMillis.shift(); + expect(date).toEqual(millis === null ? null : new Date(millis!)); + } + }); +}); + +const expectedMillis32 = () => [ + 165247430400000, 34582809600000, 232604524800000, null, + 199808812800000, 165646771200000, 209557238400000, null +]; + +const expectedMillis64 = () => [ + 27990830234011, -41278585914325, 12694624797111, + null, null, 10761360520213, null, 1394015437000 +]; + +const test_data = { + 'schema': { + 'fields': [ + { + 'name': 'f0', + 'type': { + 'name': 'date', + 'unit': 'DAY' + }, + 'nullable': true, + 'children': [] + }, + { + 'name': 'f1', + 'type': { + 'name': 'date', + 'unit': 'MILLISECOND' + }, + 'nullable': true, + 'children': [] + } + ] + }, + 'batches': [ + { + 'count': 8, + 'columns': [ + { + 'name': 'f0', + 'count': 8, + 'VALIDITY': [1, 1, 1, 0, 1, 1, 1, 0], + 'DATA': [1912586, 400264, 2692182, 2163746, 2312602, 1917208, 2425431] + }, + { + 'name': 'f1', + 'count': 8, + 'VALIDITY': [1, 1, 1, 0, 0, 1, 0, 1], + 'DATA': [ + 27990830234011, + -41278585914325, + 12694624797111, + -38604948562547, + -37802308043516, + 10761360520213, + -25129181633384, + 1394015437000 // <-- the tricky one + ] + } + ] + } + ] +}; diff --git a/src/arrow/js/test/unit/vector/numeric-vector-tests.ts b/src/arrow/js/test/unit/vector/numeric-vector-tests.ts new file mode 100644 index 000000000..61418c431 --- /dev/null +++ b/src/arrow/js/test/unit/vector/numeric-vector-tests.ts @@ -0,0 +1,616 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* eslint-disable jest/no-identical-title */ + +import { + util, + Data, Vector, + Float, Float16, Float32, Float64, + Int, Int8, Int16, Int32, Int64, Uint8, Uint16, Uint32, Uint64, + FloatVector, Float16Vector, Float32Vector, Float64Vector, + IntVector, Int8Vector, Int16Vector, Int32Vector, Int64Vector, + Uint8Vector, Uint16Vector, Uint32Vector, Uint64Vector, +} from 'apache-arrow'; + +const { float64ToUint16, uint16ToFloat64 } = util; +import { VectorType as V } from 'apache-arrow/interfaces'; +import { TypedArray, TypedArrayConstructor } from 'apache-arrow/interfaces'; +import { BigIntArray, BigIntArrayConstructor } from 'apache-arrow/interfaces'; + +const { joinUint8Arrays, BN } = util; +const uint16ToFloat64Array = (b: ArrayBuffer) => new Float64Array([...new Uint16Array(b)].map(uint16ToFloat64)); +const randomBytes = (n: number) => new Uint16Array([ + ...Uint16Array.from([0, 65535]), + ...Uint16Array.from({ length: (n / 2) - 2 }, () => (Math.random() * 65536) | 0), +]).buffer; +const toBigNumsArray = (values: Int32Array | Uint32Array) => { + const array = new Array(values.length * 0.5); + for (let i = -1, n = values.length * 0.5; ++i < n;) { + array[i] = BN.new(values.subarray(i * 2, i * 2 + 2))[Symbol.toPrimitive](); + } + return array; +}; + +const testValueBuffers = Array.from({ length: 5 }, () => randomBytes(64)); +const testValuesBuffer = joinUint8Arrays(testValueBuffers.map((b) => new Uint8Array(b)))[0].buffer; + +const checkType = <T, R extends T>(Ctor: new (...args: any) => T, inst: R) => expect(inst).toBeInstanceOf(Ctor); +const valuesArray = <T extends TypedArray>(ArrayType: TypedArrayConstructor<T>) => [...valuesTyped<T>(ArrayType)]; +const valuesArray64 = <T extends TypedArray>(ArrayType: TypedArrayConstructor<T>) => { + const typed = valuesTyped<T>(ArrayType); + const array = new Array(typed.length * 0.5); + for (let i = -1, n = array.length; ++i < n;) { + // Interleave regular Arrays and TypedArrays to cover more surface area + array[i] = i % 2 === 0 + ? [...typed.subarray(i * 2, (i + 1) * 2)] + : typed.subarray(i * 2, (i + 1) * 2); + } + return array; +}; +const valuesTyped = <T extends TypedArray>(ArrayType: TypedArrayConstructor<T>) => new ArrayType(testValuesBuffer); +const bigIntValuesTyped = <T extends BigIntArray>(ArrayType: BigIntArrayConstructor<T>) => new ArrayType(testValuesBuffer); +const bigIntValuesArray = <T extends BigIntArray>(ArrayType: BigIntArrayConstructor<T>) => [...bigIntValuesTyped<T>(ArrayType)]; + +describe(`FloatVector`, () => { + + describe(`FloatVector.from infers the type from the input TypedArray`, () => { + + const u16s = valuesTyped(Uint16Array).map((x) => float64ToUint16(uint16ToFloat64(x))); + const f16s = valuesArray(Uint16Array).map(uint16ToFloat64); + const f32s = valuesTyped(Float32Array); + const f64s = valuesTyped(Float64Array); + const f16Vec = FloatVector.from(u16s); + const f32Vec = FloatVector.from(valuesTyped(Float32Array)); + const f64Vec = FloatVector.from(valuesTyped(Float64Array)); + + // test strong typing at compile-time + test(`return type is correct`, () => checkType(Float16Vector, f16Vec)); + test(`return type is correct`, () => checkType(Float32Vector, f32Vec)); + test(`return type is correct`, () => checkType(Float64Vector, f64Vec)); + test(`throws on bad input`, () => { + expect(() => FloatVector.from(<any> {})).toThrow('Unrecognized FloatVector input'); + }); + + testAndValidateVector(f16Vec, u16s, f16s); + testAndValidateVector(f32Vec, f32s); + testAndValidateVector(f64Vec, f64s); + }); + + describe(`FloatVector.from casts the input values to the correct float type`, () => { + + const u16s = valuesTyped(Uint16Array).map((x) => float64ToUint16(uint16ToFloat64(x))); + const f16s = valuesArray(Uint16Array).map(uint16ToFloat64); + const f16Vec_ = FloatVector.from(u16s); + + const f16Vec = Float16Vector.from(f16Vec_); + const f32Vec = Float32Vector.from(f16Vec_); + const f64Vec = Float64Vector.from(f16Vec_); + + // test strong typing at compile-time + test(`return type is correct`, () => checkType(Float16Vector, f16Vec)); + test(`return type is correct`, () => checkType(Float32Vector, f32Vec)); + test(`return type is correct`, () => checkType(Float64Vector, f64Vec)); + + testAndValidateVector(f16Vec, u16s, f16s); + testAndValidateVector(f32Vec, Float32Array.from(f16s)); + testAndValidateVector(f64Vec, Float64Array.from(f16s)); + }); + + describe(`Float16Vector`, () => { + testFloatVector(Float16, valuesArray(Uint16Array).map(uint16ToFloat64)); + describe(`Float16Vector.from accepts regular Arrays`, () => { + const u16s = valuesTyped(Uint16Array).map((x) => float64ToUint16(uint16ToFloat64(x))); + const f16s = valuesArray(Uint16Array).map(uint16ToFloat64); + const vector = Float16Vector.from(f16s); + test(`return type is correct`, () => checkType(Float16Vector, vector)); + testAndValidateVector(vector, u16s, f16s); + }); + describe(`Float16Vector.from accepts Uint16Arrays`, () => { + const u16s = valuesTyped(Uint16Array).map((x) => float64ToUint16(uint16ToFloat64(x))); + const f16s = valuesArray(Uint16Array).map(uint16ToFloat64); + const vector = Float16Vector.from(u16s); + test(`return type is correct`, () => checkType(Float16Vector, vector)); + testAndValidateVector(vector, u16s, f16s); + }); + }); + describe(`Float32Vector`, () => { + testFloatVector(Float32); + describe(`Float32Vector.from accepts regular Arrays`, () => { + const values = valuesArray(Float32Array); + const vector = Float32Vector.from(values); + testAndValidateVector(vector, valuesTyped(Float32Array), values); + test(`return type is correct`, () => checkType(Float32Vector, vector)); + }); + }); + describe(`Float64Vector`, () => { + testFloatVector(Float64); + describe(`Float64Vector.from accepts regular Arrays`, () => { + const values = valuesArray(Float64Array); + const vector = Float64Vector.from(values); + testAndValidateVector(vector, valuesTyped(Float64Array), values); + test(`return type is correct`, () => checkType(Float64Vector, vector)); + }); + }); +}); + +describe(`IntVector`, () => { + + describe(`IntVector.from infers the type from the input TypedArray`, () => { + + const i8s = valuesTyped(Int8Array); + const i16s = valuesTyped(Int16Array); + const i32s = valuesTyped(Int32Array); + const i64s = valuesTyped(Int32Array); + const u8s = valuesTyped(Uint8Array); + const u16s = valuesTyped(Uint16Array); + const u32s = valuesTyped(Uint32Array); + const u64s = valuesTyped(Uint32Array); + const i8Vec = IntVector.from(i8s); + const i16Vec = IntVector.from(i16s); + const i32Vec = IntVector.from(i32s); + const i64Vec = IntVector.from(i64s, true); + const u8Vec = IntVector.from(u8s); + const u16Vec = IntVector.from(u16s); + const u32Vec = IntVector.from(u32s); + const u64Vec = IntVector.from(u64s, true); + + // test strong typing at compile-time + test(`return type is correct`, () => checkType(Int8Vector, i8Vec)); + test(`return type is correct`, () => checkType(Int16Vector, i16Vec)); + test(`return type is correct`, () => checkType(Int32Vector, i32Vec)); + test(`return type is correct`, () => checkType(Int64Vector, i64Vec)); + test(`return type is correct`, () => checkType(Uint8Vector, u8Vec)); + test(`return type is correct`, () => checkType(Uint16Vector, u16Vec)); + test(`return type is correct`, () => checkType(Uint32Vector, u32Vec)); + test(`return type is correct`, () => checkType(Uint64Vector, u64Vec)); + test(`throws on bad input`, () => { + expect(() => IntVector.from(<any> {})).toThrow('Unrecognized IntVector input'); + }); + + const bigI64s = BigInt64Array.from(toBigNumsArray(i64s)); + const bigU64s = BigUint64Array.from(toBigNumsArray(u64s)); + + testAndValidateVector(i8Vec, i8s); + testAndValidateVector(i16Vec, i16s); + testAndValidateVector(i32Vec, i32s); + // This tests when values are represented as pairs of lo, hi + testAndValidateVector(i64Vec, i64s); + // This tests when values are represented as native JS bigints + testAndValidateVector(i64Vec, i64s, [...bigI64s]); + testAndValidateVector(u8Vec, u8s); + testAndValidateVector(u16Vec, u16s); + testAndValidateVector(u32Vec, u32s); + // This tests when values are represented as pairs of lo, hi + testAndValidateVector(u64Vec, u64s); + // This tests when values are represented as native JS bigints + testAndValidateVector(u64Vec, u64s, [...bigU64s]); + }); + + describe('IntVector.from casts the input values to the correct integer type', () => { + + const i8s = valuesTyped(Int8Array); + const i16s = valuesTyped(Int16Array); + const i32s = valuesTyped(Int32Array); + const i64s = valuesTyped(Int32Array); + const u8s = valuesTyped(Uint8Array); + const u16s = valuesTyped(Uint16Array); + const u32s = valuesTyped(Uint32Array); + const u64s = valuesTyped(Uint32Array); + const i8Vec_ = IntVector.from(i8s); + const i16Vec_ = IntVector.from(i16s); + const i32Vec_ = IntVector.from(i32s); + const i64Vec_ = IntVector.from(i64s, true); + const u8Vec_ = IntVector.from(u8s); + const u16Vec_ = IntVector.from(u16s); + const u32Vec_ = IntVector.from(u32s); + const u64Vec_ = IntVector.from(u64s, true); + + // Convert from a Vector of the opposite sign + const i8Vec = Int8Vector.from(u8Vec_); + const i16Vec = Int16Vector.from(u16Vec_); + const i32Vec = Int32Vector.from(u32Vec_); + const i64Vec = Int64Vector.from(u64Vec_); + const u8Vec = Uint8Vector.from(i8Vec_); + const u16Vec = Uint16Vector.from(i16Vec_); + const u32Vec = Uint32Vector.from(i32Vec_); + const u64Vec = Uint64Vector.from(i64Vec_); + + // test strong typing at compile-time + test(`return type is correct`, () => checkType(Int8Vector, i8Vec)); + test(`return type is correct`, () => checkType(Int16Vector, i16Vec)); + test(`return type is correct`, () => checkType(Int32Vector, i32Vec)); + test(`return type is correct`, () => checkType(Int64Vector, i64Vec)); + test(`return type is correct`, () => checkType(Uint8Vector, u8Vec)); + test(`return type is correct`, () => checkType(Uint16Vector, u16Vec)); + test(`return type is correct`, () => checkType(Uint32Vector, u32Vec)); + test(`return type is correct`, () => checkType(Uint64Vector, u64Vec)); + + const bigI64s = BigInt64Array.from(toBigNumsArray(u64s)); + const bigU64s = BigUint64Array.from(toBigNumsArray(i64s)); + + testAndValidateVector(i8Vec, Int8Array.from(u8s)); + testAndValidateVector(i16Vec, Int16Array.from(u16s)); + testAndValidateVector(i32Vec, Int32Array.from(u32s)); + // This tests when values are represented as pairs of lo, hi + testAndValidateVector(i64Vec, new Int32Array(bigI64s.buffer)); + // This tests when values are represented as native JS bigints + testAndValidateVector(i64Vec, new Int32Array(bigI64s.buffer), [...bigI64s]); + testAndValidateVector(u8Vec, Uint8Array.from(i8s)); + testAndValidateVector(u16Vec, Uint16Array.from(i16s)); + testAndValidateVector(u32Vec, Uint32Array.from(i32s)); + // This tests when values are represented as pairs of lo, hi + testAndValidateVector(u64Vec, new Uint32Array(bigU64s.buffer)); + // This tests when values are represented as native JS bigints + testAndValidateVector(u64Vec, new Uint32Array(bigU64s.buffer), [...bigU64s]); + }); + + describe(`Int8Vector`, () => { + testIntVector(Int8); + describe(`Int8Vector.from accepts regular Arrays`, () => { + const values = valuesArray(Int8Array); + const vector = Int8Vector.from(values); + testAndValidateVector(vector, valuesTyped(Int8Array), values); + test(`return type is correct`, () => checkType(Int8Vector, vector)); + }); + }); + describe(`Int16Vector`, () => { + testIntVector(Int16); + describe(`Int16Vector.from accepts regular Arrays`, () => { + const values = valuesArray(Int16Array); + const vector = Int16Vector.from(values); + testAndValidateVector(vector, valuesTyped(Int16Array), values); + test(`return type is correct`, () => checkType(Int16Vector, vector)); + }); + }); + describe(`Int32Vector`, () => { + testIntVector(Int32); + describe(`Int32Vector.from accepts regular Arrays`, () => { + const values = valuesArray(Int32Array); + const vector = Int32Vector.from(values); + testAndValidateVector(vector, valuesTyped(Int32Array), values); + test(`return type is correct`, () => checkType(Int32Vector, vector)); + }); + }); + describe(`Int64Vector`, () => { + testIntVector(Int64); + testIntVector(Int64, bigIntValuesArray(BigInt64Array)); + describe(`Int64Vector.from accepts regular Arrays`, () => { + const values = valuesArray64(Int32Array); + const vector = Int64Vector.from(values); + testAndValidateVector(vector, valuesTyped(Int32Array), values); + testAndValidateVector(vector, valuesTyped(Int32Array), bigIntValuesArray(BigInt64Array)); + test(`return type is correct`, () => checkType(Int64Vector, vector)); + }); + }); + describe(`Uint8Vector`, () => { + testIntVector(Uint8); + describe(`Uint8Vector.from accepts regular Arrays`, () => { + const values = valuesArray(Uint8Array); + const vector = Uint8Vector.from(values); + testAndValidateVector(vector, valuesTyped(Uint8Array), values); + test(`return type is correct`, () => checkType(Uint8Vector, vector)); + }); + }); + describe(`Uint16Vector`, () => { + testIntVector(Uint16); + describe(`Uint16Vector.from accepts regular Arrays`, () => { + const values = valuesArray(Uint16Array); + const vector = Uint16Vector.from(values); + testAndValidateVector(vector, valuesTyped(Uint16Array), values); + test(`return type is correct`, () => checkType(Uint16Vector, vector)); + }); + }); + describe(`Uint32Vector`, () => { + testIntVector(Uint32); + describe(`Uint32Vector.from accepts regular Arrays`, () => { + const values = valuesArray(Uint32Array); + const vector = Uint32Vector.from(values); + testAndValidateVector(vector, valuesTyped(Uint32Array), values); + test(`return type is correct`, () => checkType(Uint32Vector, vector)); + }); + }); + describe(`Uint64Vector`, () => { + testIntVector(Uint64); + testIntVector(Uint64, bigIntValuesArray(BigUint64Array)); + describe(`Uint64Vector.from accepts regular Arrays`, () => { + const values = valuesArray64(Uint32Array); + const vector = Uint64Vector.from(values); + testAndValidateVector(vector, valuesTyped(Uint32Array), values); + testAndValidateVector(vector, valuesTyped(Uint32Array), bigIntValuesArray(BigUint64Array)); + test(`return type is correct`, () => checkType(Uint64Vector, vector)); + }); + }); +}); + +function testIntVector<T extends Int>(DataType: new () => T, values?: Array<any>) { + + const type = new DataType(); + const ArrayType = type.ArrayType; + const stride = type.bitWidth < 64 ? 1 : 2; + + const typed = valuesTyped(ArrayType); + const jsArray = values || [...typed]; + const vector = Vector.new(Data.Int(type, 0, typed.length / stride, 0, null, typed)); + const chunked = testValueBuffers.map((b) => new ArrayType(b)) + .map((b) => Vector.new(Data.Int(type, 0, b.length / stride, 0, null, b))) + .reduce((v: any, v2) => v.concat(v2)); + + const vectorBegin = (vector.length * .25) | 0; + const vectorEnd = (vector.length * .75) | 0; + const typedBegin = vectorBegin * (typed.length / vector.length); + const typedEnd = vectorEnd * (typed.length / vector.length); + const jsArrayBegin = vectorBegin * (jsArray.length / vector.length); + const jsArrayEnd = vectorEnd * (jsArray.length / vector.length); + + const combos = [[`vector`, vector], [`chunked`, chunked]] as [string, V<T>][]; + combos.forEach(([chunksType, vector]) => { + describe(chunksType, () => { + // test base case no slicing + describe(`base case no slicing`, () => { testAndValidateVector(vector, typed, jsArray); }); + // test slicing without args + describe(`slicing without args`, () => { testAndValidateVector(vector.slice(), typed.slice(), jsArray.slice()); }); + // test slicing the middle half + describe(`slice the middle half`, () => { + testAndValidateVector( + vector.slice(vectorBegin, vectorEnd), + typed.slice(typedBegin, typedEnd), + jsArray.slice(jsArrayBegin, jsArrayEnd) + ); + }); + // test splicing out the middle half + describe(`splicing out the middle half`, () => { + testAndValidateVector( + vector.slice(0, vectorBegin).concat(vector.slice(vectorEnd)), + new ArrayType([...typed.slice(0, typedBegin), ...typed.slice(typedEnd)]), + [...jsArray.slice(0, jsArrayBegin), ...jsArray.slice(jsArrayEnd)] + ); + }); + }); + }); +} + +function testFloatVector<T extends Float>(DataType: new () => T, values?: Array<any>) { + + const type = new DataType(); + const ArrayType = type.ArrayType; + + const typed = valuesTyped(ArrayType); + const jsArray = values || [...typed]; + const vector = Vector.new(Data.Float(type, 0, typed.length, 0, null, typed)); + const chunked = testValueBuffers.map((b) => new ArrayType(b)) + .map((b) => Vector.new(Data.Float(type, 0, b.length, 0, null, b))) + .reduce((v: any, v2) => v.concat(v2)); + + const begin = (vector.length * .25) | 0; + const end = (vector.length * .75) | 0; + const combos = [[`vector`, vector], [`chunked`, chunked]] as [string, V<T>][]; + + combos.forEach(([chunksType, vector]) => { + describe(chunksType, () => { + // test base case no slicing + describe(`base case no slicing`, () => { testAndValidateVector(vector, typed, jsArray); }); + // test slicing without args + describe(`slicing without args`, () => { testAndValidateVector(vector.slice(), typed.slice(), jsArray.slice()); }); + // test slicing the middle half + describe(`slice the middle half`, () => { + testAndValidateVector( + vector.slice(begin, end), + typed.slice(begin, end), + jsArray.slice(begin, end) + ); + }); + // test splicing out the middle half + describe(`splicing out the middle half`, () => { + testAndValidateVector( + vector.slice(0, begin).concat(vector.slice(end)), + new ArrayType([...typed.slice(0, begin), ...typed.slice(end)]), + [...jsArray.slice(0, begin), ...jsArray.slice(end)] + ); + }); + }); + }); +} + +function testAndValidateVector<T extends Int | Float>(vector: Vector<T>, typed: T['TArray'], values: any[] = [...typed]) { + gets_expected_values(vector, typed, values); + iterates_expected_values(vector, typed, values); + indexof_returns_expected_values(vector, typed, values); + slice_returns_a_typedarray(vector); + slices_the_entire_array(vector, typed); + slices_from_minus_20_to_length(vector, typed); + slices_from_0_to_minus_20(vector, typed); + slices_the_array_from_0_to_length_minus_20(vector, typed); + slices_the_array_from_0_to_length_plus_20(vector, typed); +} + +function gets_expected_values<T extends Int | Float>(vector: Vector<T>, typed: T['TArray'], values: any[] = [...typed]) { + test(`gets expected values`, () => { + expect.hasAssertions(); + let i = -1, n = vector.length; + let stride = vector.stride; + try { + if (stride === 1) { + while (++i < n) { + expect(vector.get(i)).toEqual(values[i]); + } + } else if (typeof values[0] === 'bigint') { + while (++i < n) { + const x: any = vector.get(i)!; + expect(0n + x).toEqual(values[i]); + } + } else { + const vector64 = vector as Vector<Int64 | Uint64>; + const i64 = (() => typed.subarray(stride * i, stride * (i + 1))); + while (++i < n) { + expect((vector64.get(i) as any).subarray(0, stride)).toEqual(i64()); + } + } + } catch (e) { throw new Error(`${i}: ${e}`); } + }); +} + +function iterates_expected_values<T extends Int | Float>(vector: Vector<T>, typed: T['TArray'], values: any[] = [...typed]) { + test(`iterates expected values`, () => { + expect.hasAssertions(); + let i = -1, n = vector.length; + let stride = vector.stride; + try { + if (stride === 1) { + for (let v of vector) { + expect(++i).toBeLessThan(n); + expect(v).toEqual(values[i]); + } + } else if (typeof values[0] === 'bigint') { + let x: any; + for (let v of vector) { + x = v; + expect(++i).toBeLessThan(n); + expect(0n + x).toEqual(values[i]); + } + } else { + const vector64 = vector as Vector<Int64 | Uint64>; + const i64 = (() => typed.subarray(stride * i, stride * (i + 1))); + for (let v of vector64) { + expect(++i).toBeLessThan(n); + expect((v as any).subarray(0, stride)).toEqual(i64()); + } + } + } catch (e) { throw new Error(`${i}: ${e}`); } + }); +} + +function indexof_returns_expected_values<T extends Int | Float>(vector: Vector<T>, typed: T['TArray'], values: any = [...typed]) { + test(`indexOf returns expected values`, () => { + + expect.hasAssertions(); + + const stride = vector.stride; + const BPE = vector.ArrayType.BYTES_PER_ELEMENT; + const isBigInt = typeof values[0] === 'bigint'; + const isInt64 = util.compareTypes(vector.type, new Int64()); + const isFloat16 = util.compareTypes(vector.type, new Float16()); + + // Create a few random values + let missing: any = new vector.ArrayType(randomBytes(8 * 2 * BPE)); + + // Special cases convert the values and/or missing to the + // representations that indexOf() expects to receive + + if (isFloat16) { + missing = uint16ToFloat64Array(missing); + } else if (isBigInt) { + const BigIntArray = isInt64 ? BigInt64Array : BigUint64Array; + missing = Array.from({ length: missing.length / stride }, + (_, i) => new BigIntArray(missing.buffer, BPE * stride * i, 1)[0]); + } else if (stride !== 1) { + values = Array.from({ length: typed.length / stride }, + (_, i) => typed.slice(stride * i, stride * (i + 1))); + missing = Array.from({ length: missing.length / stride }, + (_, i) => missing.slice(stride * i, stride * (i + 1))); + } + + const original = values.slice(); + // Combine with the expected values and shuffle the order + const shuffled = shuffle(values.concat([...missing])); + let i = -1, j: number, k: number, n = shuffled.length; + + try { + if (!isBigInt) { + while (++i < n) { + const search = shuffled[i]; + if (typeof search !== 'number' || !isNaN(search)) { + expect(vector.indexOf(search)).toEqual(original.indexOf(search)); + } else { + for (j = -1, k = original.length; ++j < k;) { + if (isNaN(original[j])) { break; } + } + expect(vector.indexOf(search)).toEqual(j < k ? j : -1); + } + } + } else { + // Distinguish the bigint comparisons to ensure the indexOf type signature accepts bigints + let shuffled64 = shuffled as bigint[]; + if (isInt64) { + let vector64 = (<unknown> vector) as Int64Vector; + while (++i < n) { + expect(vector64.indexOf(shuffled64[i])).toEqual(original.indexOf(shuffled64[i])); + } + } else { + let vector64 = (<unknown> vector) as Uint64Vector; + while (++i < n) { + expect(vector64.indexOf(shuffled64[i])).toEqual(original.indexOf(shuffled64[i])); + } + } + } + } catch (e) { throw new Error(`${i} (${shuffled[i]}): ${e}`); } + }); +} + +function slice_returns_a_typedarray<T extends Int | Float>(vector: Vector<T>) { + test(`slice returns a TypedArray`, () => { + expect.hasAssertions(); + expect(vector.slice().toArray()).toBeInstanceOf(vector.ArrayType); + }); +} + +function slices_the_entire_array<T extends Int | Float>(vector: Vector<T>, values: T['TArray']) { + test(`slices the entire array`, () => { + expect.hasAssertions(); + expect(vector.slice().toArray()).toEqual(values); + }); +} + +function slices_from_minus_20_to_length<T extends Int | Float>(vector: Vector<T>, values: T['TArray']) { + test(`slices from -20 to length`, () => { + expect.hasAssertions(); + expect(vector.slice(-20).toArray()).toEqual(values.slice(-(20 * vector.stride))); + }); +} + +function slices_from_0_to_minus_20<T extends Int | Float>(vector: Vector<T>, values: T['TArray']) { + test(`slices from 0 to -20`, () => { + expect.hasAssertions(); + expect(vector.slice(0, -20).toArray()).toEqual(values.slice(0, -(20 * vector.stride))); + }); +} + +function slices_the_array_from_0_to_length_minus_20 <T extends Int | Float>(vector: Vector<T>, values: T['TArray']) { + test(`slices the array from 0 to length - 20`, () => { + expect.hasAssertions(); + expect(vector.slice(0, vector.length - 20).toArray()).toEqual(values.slice(0, values.length - (20 * vector.stride))); + }); +} + +function slices_the_array_from_0_to_length_plus_20<T extends Int | Float>(vector: Vector<T>, values: T['TArray']) { + test(`slices the array from 0 to length + 20`, () => { + expect.hasAssertions(); + expect(vector.slice(0, vector.length + 20).toArray()).toEqual(values.slice(0, values.length + (20 * vector.stride))); + }); +} + +function shuffle(input: any[]) { + const result = input.slice(); + let j, tmp, i = result.length; + while (--i > 0) { + j = (Math.random() * (i + 1)) | 0; + tmp = result[i]; + result[i] = result[j]; + result[j] = tmp; + } + return result; +} diff --git a/src/arrow/js/test/unit/vector/vector-tests.ts b/src/arrow/js/test/unit/vector/vector-tests.ts new file mode 100644 index 000000000..60bff94f8 --- /dev/null +++ b/src/arrow/js/test/unit/vector/vector-tests.ts @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { + Int32, Dictionary, DateUnit, util, + Data, Vector, Utf8Vector, DateVector, DictionaryVector, +} from 'apache-arrow'; + +describe(`DateVector`, () => { + const extras = [ + new Date(2000, 0, 1), + new Date(1991, 5, 28, 12, 11, 10) + ]; + describe(`unit = MILLISECOND`, () => { + const values = [ + new Date(1989, 5, 22, 1, 2, 3), + new Date(1988, 3, 25, 4, 5, 6), + new Date(1987, 2, 24, 7, 8, 9), + new Date(2018, 4, 12, 17, 30, 0) + ]; + const vector = DateVector.from(values); + basicVectorTests(vector, values, extras); + }); + describe(`unit = DAY`, () => { + // Use UTC to ensure that dates are always at midnight + const values = [ + new Date(Date.UTC(1989, 5, 22)), + new Date(Date.UTC(1988, 3, 25)), + new Date(Date.UTC(1987, 2, 24)), + new Date(Date.UTC(2018, 4, 12)) + ]; + const vector = DateVector.from(values, DateUnit.DAY); + basicVectorTests(vector, values, extras); + }); +}); + +describe(`DictionaryVector`, () => { + + const dictionary = ['foo', 'bar', 'baz']; + const extras = ['abc', '123']; // values to search for that should NOT be found + const dictionary_vec = Utf8Vector.from(dictionary); + + const indices = Array.from({length: 50}, () => Math.random() * 3 | 0); + const validity = Array.from({ length: indices.length }, () => Math.random() > 0.2 ? true : false); + + describe(`index with nullCount == 0`, () => { + + const values = Array.from(indices).map((d) => dictionary[d]); + const vector = DictionaryVector.from(dictionary_vec, new Int32(), indices); + + basicVectorTests(vector, values, extras); + + describe(`sliced`, () => { + basicVectorTests(vector.slice(10, 20), values.slice(10,20), extras); + }); + }); + + describe(`index with nullCount > 0`, () => { + + const nullBitmap = util.packBools(validity); + const nullCount = validity.reduce((acc, d) => acc + (d ? 0 : 1), 0); + const values = Array.from(indices).map((d, i) => validity[i] ? dictionary[d] : null); + const type = new Dictionary(dictionary_vec.type, new Int32(), null, null); + const vector = Vector.new(Data.Dictionary(type, 0, indices.length, nullCount, nullBitmap, indices, dictionary_vec)); + + basicVectorTests(vector, values, ['abc', '123']); + describe(`sliced`, () => { + basicVectorTests(vector.slice(10, 20), values.slice(10,20), extras); + }); + }); +}); + +describe(`Utf8Vector`, () => { + const values = ['foo', 'bar', 'baz', 'foo bar', 'bar']; + const vector = Utf8Vector.from(values); + basicVectorTests(vector, values, ['abc', '123']); + describe(`sliced`, () => { + basicVectorTests(vector.slice(1,3), values.slice(1,3), ['foo', 'abc']); + }); +}); + +// Creates some basic tests for the given vector. +// Verifies that: +// - `get` and the native iterator return the same data as `values` +// - `indexOf` returns the same indices as `values` +function basicVectorTests(vector: Vector, values: any[], extras: any[]) { + + const n = values.length; + + test(`gets expected values`, () => { + let i = -1; + while (++i < n) { + expect(vector.get(i)).toEqual(values[i]); + } + }); + test(`iterates expected values`, () => { + expect.hasAssertions(); + let i = -1; + for (let v of vector) { + expect(++i).toBeLessThan(n); + expect(v).toEqual(values[i]); + } + }); + test(`indexOf returns expected values`, () => { + let testValues = values.concat(extras); + + for (const value of testValues) { + const actual = vector.indexOf(value); + const expected = values.indexOf(value); + expect(actual).toEqual(expected); + } + }); +} diff --git a/src/arrow/js/test/unit/visitor-tests.ts b/src/arrow/js/test/unit/visitor-tests.ts new file mode 100644 index 000000000..22b3e5ced --- /dev/null +++ b/src/arrow/js/test/unit/visitor-tests.ts @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { Field } from 'apache-arrow'; +import { Visitor } from 'apache-arrow'; +import { + DataType, Dictionary, + Bool, Null, Utf8, Binary, Decimal, FixedSizeBinary, List, FixedSizeList, Map_, Struct, + Float, Float16, Float32, Float64, + Int, Uint8, Uint16, Uint32, Uint64, Int8, Int16, Int32, Int64, + Date_, DateDay, DateMillisecond, + Interval, IntervalDayTime, IntervalYearMonth, + Time, TimeSecond, TimeMillisecond, TimeMicrosecond, TimeNanosecond, + Timestamp, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, + Union, DenseUnion, SparseUnion, +} from 'apache-arrow'; + +class BasicVisitor extends Visitor { + public type: DataType | undefined; + public visitNull <T extends Null> (type: T) { return (this.type = type); } + public visitBool <T extends Bool> (type: T) { return (this.type = type); } + public visitInt <T extends Int> (type: T) { return (this.type = type); } + public visitFloat <T extends Float> (type: T) { return (this.type = type); } + public visitUtf8 <T extends Utf8> (type: T) { return (this.type = type); } + public visitBinary <T extends Binary> (type: T) { return (this.type = type); } + public visitFixedSizeBinary <T extends FixedSizeBinary> (type: T) { return (this.type = type); } + public visitDate <T extends Date_> (type: T) { return (this.type = type); } + public visitTimestamp <T extends Timestamp> (type: T) { return (this.type = type); } + public visitTime <T extends Time> (type: T) { return (this.type = type); } + public visitDecimal <T extends Decimal> (type: T) { return (this.type = type); } + public visitList <T extends List> (type: T) { return (this.type = type); } + public visitStruct <T extends Struct> (type: T) { return (this.type = type); } + public visitUnion <T extends Union> (type: T) { return (this.type = type); } + public visitDictionary <T extends Dictionary> (type: T) { return (this.type = type); } + public visitInterval <T extends Interval> (type: T) { return (this.type = type); } + public visitFixedSizeList <T extends FixedSizeList> (type: T) { return (this.type = type); } + public visitMap <T extends Map_> (type: T) { return (this.type = type); } +} + +class FeatureVisitor extends Visitor { + public type: DataType | undefined; + public visitNull <T extends Null> (type: T) { return (this.type = type); } + public visitBool <T extends Bool> (type: T) { return (this.type = type); } + public visitInt8 <T extends Int8> (type: T) { return (this.type = type); } + public visitInt16 <T extends Int16> (type: T) { return (this.type = type); } + public visitInt32 <T extends Int32> (type: T) { return (this.type = type); } + public visitInt64 <T extends Int64> (type: T) { return (this.type = type); } + public visitUint8 <T extends Uint8> (type: T) { return (this.type = type); } + public visitUint16 <T extends Uint16> (type: T) { return (this.type = type); } + public visitUint32 <T extends Uint32> (type: T) { return (this.type = type); } + public visitUint64 <T extends Uint64> (type: T) { return (this.type = type); } + public visitFloat16 <T extends Float16> (type: T) { return (this.type = type); } + public visitFloat32 <T extends Float32> (type: T) { return (this.type = type); } + public visitFloat64 <T extends Float64> (type: T) { return (this.type = type); } + public visitUtf8 <T extends Utf8> (type: T) { return (this.type = type); } + public visitBinary <T extends Binary> (type: T) { return (this.type = type); } + public visitFixedSizeBinary <T extends FixedSizeBinary> (type: T) { return (this.type = type); } + public visitDateDay <T extends DateDay> (type: T) { return (this.type = type); } + public visitDateMillisecond <T extends DateMillisecond> (type: T) { return (this.type = type); } + public visitTimestampSecond <T extends TimestampSecond> (type: T) { return (this.type = type); } + public visitTimestampMillisecond <T extends TimestampMillisecond> (type: T) { return (this.type = type); } + public visitTimestampMicrosecond <T extends TimestampMicrosecond> (type: T) { return (this.type = type); } + public visitTimestampNanosecond <T extends TimestampNanosecond> (type: T) { return (this.type = type); } + public visitTimeSecond <T extends TimeSecond> (type: T) { return (this.type = type); } + public visitTimeMillisecond <T extends TimeMillisecond> (type: T) { return (this.type = type); } + public visitTimeMicrosecond <T extends TimeMicrosecond> (type: T) { return (this.type = type); } + public visitTimeNanosecond <T extends TimeNanosecond> (type: T) { return (this.type = type); } + public visitDecimal <T extends Decimal> (type: T) { return (this.type = type); } + public visitList <T extends List> (type: T) { return (this.type = type); } + public visitStruct <T extends Struct> (type: T) { return (this.type = type); } + public visitDenseUnion <T extends DenseUnion> (type: T) { return (this.type = type); } + public visitSparseUnion <T extends SparseUnion> (type: T) { return (this.type = type); } + public visitDictionary <T extends Dictionary> (type: T) { return (this.type = type); } + public visitIntervalDayTime <T extends IntervalDayTime> (type: T) { return (this.type = type); } + public visitIntervalYearMonth <T extends IntervalYearMonth> (type: T) { return (this.type = type); } + public visitFixedSizeList <T extends FixedSizeList> (type: T) { return (this.type = type); } + public visitMap <T extends Map_> (type: T) { return (this.type = type); } +} + +describe('Visitor', () => { + + describe('uses the base methods when no feature methods are implemented', () => { + test(`visits Null types`, () => validateBasicVisitor(new Null())); + test(`visits Bool types`, () => validateBasicVisitor(new Bool())); + test(`visits Int types`, () => validateBasicVisitor(new Int(true, 32))); + test(`visits Float types`, () => validateBasicVisitor(new Float(0))); + test(`visits Utf8 types`, () => validateBasicVisitor(new Utf8())); + test(`visits Binary types`, () => validateBasicVisitor(new Binary())); + test(`visits FixedSizeBinary types`, () => validateBasicVisitor(new FixedSizeBinary(128))); + test(`visits Date types`, () => validateBasicVisitor(new Date_(0))); + test(`visits Timestamp types`, () => validateBasicVisitor(new Timestamp(0, 'UTC'))); + test(`visits Time types`, () => validateBasicVisitor(new Time(0, 64))); + test(`visits Decimal types`, () => validateBasicVisitor(new Decimal(2, 9))); + test(`visits List types`, () => validateBasicVisitor(new List(null as any))); + test(`visits Struct types`, () => validateBasicVisitor(new Struct([] as any[]))); + test(`visits Union types`, () => validateBasicVisitor(new Union(0, [] as any[], [] as any[]))); + test(`visits Dictionary types`, () => validateBasicVisitor(new Dictionary(null as any, null as any))); + test(`visits Interval types`, () => validateBasicVisitor(new Interval(0))); + test(`visits FixedSizeList types`, () => validateBasicVisitor(new FixedSizeList(2, null as any))); + test(`visits Map types`, () => validateBasicVisitor(new Map_(new Field('', new Struct<{ key: Int; value: Int }>([] as any[]))))); + function validateBasicVisitor<T extends DataType>(type: T) { + const visitor = new BasicVisitor(); + const result = visitor.visit(type); + expect(result).toBe(type); + expect(visitor.type).toBe(type); + } + }); + + describe(`uses the feature methods instead of the base methods when they're implemented`, () => { + + test(`visits Null types`, () => validateFeatureVisitor(new Null())); + test(`visits Bool types`, () => validateFeatureVisitor(new Bool())); + test(`visits Int8 types`, () => validateFeatureVisitor(new Int8())); + test(`visits Int16 types`, () => validateFeatureVisitor(new Int16())); + test(`visits Int32 types`, () => validateFeatureVisitor(new Int32())); + test(`visits Int64 types`, () => validateFeatureVisitor(new Int64())); + test(`visits Uint8 types`, () => validateFeatureVisitor(new Uint8())); + test(`visits Uint16 types`, () => validateFeatureVisitor(new Uint16())); + test(`visits Uint32 types`, () => validateFeatureVisitor(new Uint32())); + test(`visits Uint64 types`, () => validateFeatureVisitor(new Uint64())); + test(`visits Float16 types`, () => validateFeatureVisitor(new Float16())); + test(`visits Float32 types`, () => validateFeatureVisitor(new Float32())); + test(`visits Float64 types`, () => validateFeatureVisitor(new Float64())); + test(`visits Utf8 types`, () => validateFeatureVisitor(new Utf8())); + test(`visits Binary types`, () => validateFeatureVisitor(new Binary())); + test(`visits FixedSizeBinary types`, () => validateFeatureVisitor(new FixedSizeBinary(128))); + test(`visits DateDay types`, () => validateFeatureVisitor(new DateDay())); + test(`visits DateMillisecond types`, () => validateFeatureVisitor(new DateMillisecond())); + test(`visits TimestampSecond types`, () => validateFeatureVisitor(new TimestampSecond())); + test(`visits TimestampMillisecond types`, () => validateFeatureVisitor(new TimestampMillisecond())); + test(`visits TimestampMicrosecond types`, () => validateFeatureVisitor(new TimestampMicrosecond())); + test(`visits TimestampNanosecond types`, () => validateFeatureVisitor(new TimestampNanosecond())); + test(`visits TimeSecond types`, () => validateFeatureVisitor(new TimeSecond())); + test(`visits TimeMillisecond types`, () => validateFeatureVisitor(new TimeMillisecond())); + test(`visits TimeMicrosecond types`, () => validateFeatureVisitor(new TimeMicrosecond())); + test(`visits TimeNanosecond types`, () => validateFeatureVisitor(new TimeNanosecond())); + test(`visits Decimal types`, () => validateFeatureVisitor(new Decimal(2, 9))); + test(`visits List types`, () => validateFeatureVisitor(new List(null as any))); + test(`visits Struct types`, () => validateFeatureVisitor(new Struct([] as any[]))); + test(`visits DenseUnion types`, () => validateFeatureVisitor(new DenseUnion([] as any[], [] as any[]))); + test(`visits SparseUnion types`, () => validateFeatureVisitor(new SparseUnion([] as any[], [] as any[]))); + test(`visits Dictionary types`, () => validateFeatureVisitor(new Dictionary(null as any, null as any))); + test(`visits IntervalDayTime types`, () => validateFeatureVisitor(new IntervalDayTime())); + test(`visits IntervalYearMonth types`, () => validateFeatureVisitor(new IntervalYearMonth())); + test(`visits FixedSizeList types`, () => validateFeatureVisitor(new FixedSizeList(2, null as any))); + test(`visits Map types`, () => validateFeatureVisitor(new Map_(new Field('', new Struct<{ key: Int; value: Int }>([] as any[]))))); + + function validateFeatureVisitor<T extends DataType>(type: T) { + const visitor = new FeatureVisitor(); + const result = visitor.visit(type); + expect(result).toBe(type); + expect(visitor.type).toBe(type); + } + }); +}); diff --git a/src/arrow/js/tsconfig.json b/src/arrow/js/tsconfig.json new file mode 100644 index 000000000..c1e02ca01 --- /dev/null +++ b/src/arrow/js/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "./tsconfig/tsconfig.base.json", + "formatCodeOptions": { + "tabSize": 2, + "indentSize": 2 + }, + "compilerOptions": { + "target": "esnext", + "module": "commonjs", + "noEmit": true, + "esModuleInterop": true, + "baseUrl": "./", + "paths": { + "apache-arrow": ["src/Arrow.node"], + "apache-arrow/*": ["src/*"] + } + }, + "include": ["src/**/*.ts", "test/**/*.ts", "perf/**/*.ts"] +} diff --git a/src/arrow/js/tsconfig/tsconfig.base.json b/src/arrow/js/tsconfig/tsconfig.base.json new file mode 100644 index 000000000..8ee0d98f6 --- /dev/null +++ b/src/arrow/js/tsconfig/tsconfig.base.json @@ -0,0 +1,50 @@ +{ + "exclude": ["../node_modules"], + "include": ["../src/**/*.ts"], + "compileOnSave": false, + "compilerOptions": { + "baseUrl": "./", + "paths": { + "apache-arrow/*": ["src/*"] + }, + + /* Basic stuff */ + "moduleResolution": "node", + "lib": ["dom", "esnext", "esnext.asynciterable"], + + /* Control what is emitted */ + "declaration": true, + "declarationMap": true, + "noEmitOnError": true, + "removeComments": false, + "noErrorTruncation": true, + "downlevelIteration": true, + + /* Create inline sourcemaps with sources */ + "sourceMap": false, + "inlineSources": true, + "inlineSourceMap": true, + + /* The most restrictive settings possible */ + "strict": true, + "noImplicitAny": true, + "noImplicitThis": true, + "alwaysStrict": true, + "strictBindCallApply": true, + "strictNullChecks": true, + "strictFunctionTypes": true, + "strictPropertyInitialization": true, + + "skipLibCheck": false, + "importHelpers": true, + "noEmitHelpers": true, + "noUnusedLocals": true, + "noImplicitReturns": true, + "noUnusedParameters": true, + "allowUnusedLabels": false, + "allowUnreachableCode": false, + "noStrictGenericChecks": false, + "noFallthroughCasesInSwitch": true, + "forceConsistentCasingInFileNames": true + } +} diff --git a/src/arrow/js/tsconfig/tsconfig.bin.cjs.json b/src/arrow/js/tsconfig/tsconfig.bin.cjs.json new file mode 100644 index 000000000..e9671810a --- /dev/null +++ b/src/arrow/js/tsconfig/tsconfig.bin.cjs.json @@ -0,0 +1,12 @@ +// Compiler configuration to build the ES5 CommonJS bin files +{ + "extends": "./tsconfig.base.json", + "exclude": ["../node_modules"], + "include": ["../src/bin/*.ts"], + "compilerOptions": { + "target": "esnext", + "module": "commonjs", + "declaration": false, + "declarationMap": false + } +} diff --git a/src/arrow/js/tsconfig/tsconfig.docs.json b/src/arrow/js/tsconfig/tsconfig.docs.json new file mode 100644 index 000000000..722838f5b --- /dev/null +++ b/src/arrow/js/tsconfig/tsconfig.docs.json @@ -0,0 +1,8 @@ +// Compiler configuration to build the docs +{ + "extends": "./tsconfig.base.json", + "include": ["../src/**/*.ts"], + "compilerOptions": { + "target": "esnext" + } +} diff --git a/src/arrow/js/tsconfig/tsconfig.es2015.cjs.json b/src/arrow/js/tsconfig/tsconfig.es2015.cjs.json new file mode 100644 index 000000000..92f05dd1c --- /dev/null +++ b/src/arrow/js/tsconfig/tsconfig.es2015.cjs.json @@ -0,0 +1,8 @@ +// Compiler configuration to build the ES2015 CommonJS target +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "es2015", + "module": "commonjs" + } +} diff --git a/src/arrow/js/tsconfig/tsconfig.es2015.cls.json b/src/arrow/js/tsconfig/tsconfig.es2015.cls.json new file mode 100644 index 000000000..7cc364b36 --- /dev/null +++ b/src/arrow/js/tsconfig/tsconfig.es2015.cls.json @@ -0,0 +1,12 @@ +// Compiler configuration to build the ES2015 Closure Compiler target +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "es2015", + "declaration": false, + "declarationMap": false, + "noEmitHelpers": true, + "importHelpers": false + } +} diff --git a/src/arrow/js/tsconfig/tsconfig.es2015.esm.json b/src/arrow/js/tsconfig/tsconfig.es2015.esm.json new file mode 100644 index 000000000..c56b97263 --- /dev/null +++ b/src/arrow/js/tsconfig/tsconfig.es2015.esm.json @@ -0,0 +1,8 @@ +// Compiler configuration to build the ES2015 ESModules target +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "es2015", + "module": "es2015" + } +} diff --git a/src/arrow/js/tsconfig/tsconfig.es5.cjs.json b/src/arrow/js/tsconfig/tsconfig.es5.cjs.json new file mode 100644 index 000000000..7c149d39d --- /dev/null +++ b/src/arrow/js/tsconfig/tsconfig.es5.cjs.json @@ -0,0 +1,8 @@ +// Compiler configuration to build the ES5 CommonJS target +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "es5", + "module": "commonjs" + } +} diff --git a/src/arrow/js/tsconfig/tsconfig.es5.cls.json b/src/arrow/js/tsconfig/tsconfig.es5.cls.json new file mode 100644 index 000000000..a03808d36 --- /dev/null +++ b/src/arrow/js/tsconfig/tsconfig.es5.cls.json @@ -0,0 +1,12 @@ +// Compiler configuration to build the ES5 Closure Compiler target +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "es2015", + "declaration": false, + "declarationMap": false, + "noEmitHelpers": true, + "importHelpers": false + } +} diff --git a/src/arrow/js/tsconfig/tsconfig.es5.esm.json b/src/arrow/js/tsconfig/tsconfig.es5.esm.json new file mode 100644 index 000000000..782c303e6 --- /dev/null +++ b/src/arrow/js/tsconfig/tsconfig.es5.esm.json @@ -0,0 +1,8 @@ +// Compiler configuration to build the ES5 ESModules target +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "es5", + "module": "es2015" + } +} diff --git a/src/arrow/js/tsconfig/tsconfig.esnext.cjs.json b/src/arrow/js/tsconfig/tsconfig.esnext.cjs.json new file mode 100644 index 000000000..fb0d2eb11 --- /dev/null +++ b/src/arrow/js/tsconfig/tsconfig.esnext.cjs.json @@ -0,0 +1,8 @@ +// Compiler configuration to build the ESNext CommonJS target +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "commonjs" + } +} diff --git a/src/arrow/js/tsconfig/tsconfig.esnext.cls.json b/src/arrow/js/tsconfig/tsconfig.esnext.cls.json new file mode 100644 index 000000000..dc35c3f88 --- /dev/null +++ b/src/arrow/js/tsconfig/tsconfig.esnext.cls.json @@ -0,0 +1,12 @@ +// Compiler configuration to build the ESNext Closure Compiler target +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "es2015", + "declaration": false, + "declarationMap": false, + "noEmitHelpers": true, + "importHelpers": false + } +} diff --git a/src/arrow/js/tsconfig/tsconfig.esnext.esm.json b/src/arrow/js/tsconfig/tsconfig.esnext.esm.json new file mode 100644 index 000000000..6701c8e13 --- /dev/null +++ b/src/arrow/js/tsconfig/tsconfig.esnext.esm.json @@ -0,0 +1,8 @@ +// Compiler configuration to build the ESNext ESModules target +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "target": "esnext", + "module": "es2015" + } +} diff --git a/src/arrow/js/typedoc.js b/src/arrow/js/typedoc.js new file mode 100644 index 000000000..3512c01f2 --- /dev/null +++ b/src/arrow/js/typedoc.js @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +module.exports = { + entryPoints: ['src/Arrow.dom.ts', 'src/Arrow.node.ts'], + out: 'doc', + name: 'Apache Arrow', + tsconfig: 'tsconfig/tsconfig.docs.json', + excludePrivate: true, + excludeProtected: true, + excludeExternals: true, + includeVersion: true, + exclude: [ + 'src/fb/*.ts', + 'src/bin/*.ts' + ] +}; diff --git a/src/arrow/js/yarn.lock b/src/arrow/js/yarn.lock new file mode 100644 index 000000000..3c77aeed0 --- /dev/null +++ b/src/arrow/js/yarn.lock @@ -0,0 +1,9521 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@arrows/array@^1.4.0": + version "1.4.1" + resolved "https://registry.yarnpkg.com/@arrows/array/-/array-1.4.1.tgz#a6580a08cee219755ca9a8eb14e956d3c29a5508" + integrity sha512-MGYS8xi3c4tTy1ivhrVntFvufoNzje0PchjEz6G/SsWRgUKxL4tKwS6iPdO8vsaJYldagAeWMd5KRD0aX3Q39g== + dependencies: + "@arrows/composition" "^1.2.2" + +"@arrows/composition@^1.0.0", "@arrows/composition@^1.2.2": + version "1.2.2" + resolved "https://registry.yarnpkg.com/@arrows/composition/-/composition-1.2.2.tgz#d0a213cac8f8c36c1c75856a1e6ed940c27e9169" + integrity sha512-9fh1yHwrx32lundiB3SlZ/VwuStPB4QakPsSLrGJFH6rCXvdrd060ivAZ7/2vlqPnEjBkPRRXOcG1YOu19p2GQ== + +"@arrows/dispatch@^1.0.2": + version "1.0.3" + resolved "https://registry.yarnpkg.com/@arrows/dispatch/-/dispatch-1.0.3.tgz#c4c06260f89e9dd4ce280df3712980aa2f3de976" + integrity sha512-v/HwvrFonitYZM2PmBlAlCqVqxrkIIoiEuy5bQgn0BdfvlL0ooSBzcPzTMrtzY8eYktPyYcHg8fLbSgyybXEqw== + dependencies: + "@arrows/composition" "^1.2.2" + +"@arrows/error@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@arrows/error/-/error-1.0.2.tgz#4e68036f901118ba6f1de88656ef6be49e650414" + integrity sha512-yvkiv1ay4Z3+Z6oQsUkedsQm5aFdyPpkBUQs8vejazU/RmANABx6bMMcBPPHI4aW43VPQmXFfBzr/4FExwWTEA== + +"@arrows/multimethod@^1.1.6": + version "1.1.7" + resolved "https://registry.yarnpkg.com/@arrows/multimethod/-/multimethod-1.1.7.tgz#bc7c26c3aa7703fc967e65da4f00718b1428eb4a" + integrity sha512-EjHD3XuGAV4G28rm7mu8k7zQJh/EOizh104/p9i2ofGcnL5mgKONFH/Bq6H3SJjM+WDAlKcR9WBpNhaAKCnH2g== + dependencies: + "@arrows/array" "^1.4.0" + "@arrows/composition" "^1.2.2" + "@arrows/error" "^1.0.2" + fast-deep-equal "^3.1.1" + +"@babel/code-frame@7.12.11": + version "7.12.11" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.11.tgz#f4ad435aa263db935b8f10f2c552d23fb716a63f" + integrity sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw== + dependencies: + "@babel/highlight" "^7.10.4" + +"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.14.5.tgz#23b08d740e83f49c5e59945fbf1b43e80bbf4edb" + integrity sha512-9pzDqyc6OLDaqe+zbACgFkb6fKMNG6CObKpnYXChRsvYGyEdc7CA2BaqeOM+vOtCS5ndmJicPJhKAwYRI6UfFw== + dependencies: + "@babel/highlight" "^7.14.5" + +"@babel/compat-data@^7.14.5": + version "7.14.7" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.14.7.tgz#7b047d7a3a89a67d2258dc61f604f098f1bc7e08" + integrity sha512-nS6dZaISCXJ3+518CWiBfEr//gHyMO02uDxBkXTKZDN5POruCnOZ1N4YBRZDCabwF8nZMWBpRxIicmXtBs+fvw== + +"@babel/core@^7.1.0", "@babel/core@^7.7.2", "@babel/core@^7.7.5": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.14.8.tgz#20cdf7c84b5d86d83fac8710a8bc605a7ba3f010" + integrity sha512-/AtaeEhT6ErpDhInbXmjHcUQXH0L0TEgscfcxk1qbOvLuKCa5aZT0SOOtDKFY96/CLROwbLSKyFor6idgNaU4Q== + dependencies: + "@babel/code-frame" "^7.14.5" + "@babel/generator" "^7.14.8" + "@babel/helper-compilation-targets" "^7.14.5" + "@babel/helper-module-transforms" "^7.14.8" + "@babel/helpers" "^7.14.8" + "@babel/parser" "^7.14.8" + "@babel/template" "^7.14.5" + "@babel/traverse" "^7.14.8" + "@babel/types" "^7.14.8" + convert-source-map "^1.7.0" + debug "^4.1.0" + gensync "^1.0.0-beta.2" + json5 "^2.1.2" + semver "^6.3.0" + source-map "^0.5.0" + +"@babel/generator@^7.14.8", "@babel/generator@^7.7.2": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.14.8.tgz#bf86fd6af96cf3b74395a8ca409515f89423e070" + integrity sha512-cYDUpvIzhBVnMzRoY1fkSEhK/HmwEVwlyULYgn/tMQYd6Obag3ylCjONle3gdErfXBW61SVTlR9QR7uWlgeIkg== + dependencies: + "@babel/types" "^7.14.8" + jsesc "^2.5.1" + source-map "^0.5.0" + +"@babel/helper-compilation-targets@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.14.5.tgz#7a99c5d0967911e972fe2c3411f7d5b498498ecf" + integrity sha512-v+QtZqXEiOnpO6EYvlImB6zCD2Lel06RzOPzmkz/D/XgQiUu3C/Jb1LOqSt/AIA34TYi/Q+KlT8vTQrgdxkbLw== + dependencies: + "@babel/compat-data" "^7.14.5" + "@babel/helper-validator-option" "^7.14.5" + browserslist "^4.16.6" + semver "^6.3.0" + +"@babel/helper-function-name@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.14.5.tgz#89e2c474972f15d8e233b52ee8c480e2cfcd50c4" + integrity sha512-Gjna0AsXWfFvrAuX+VKcN/aNNWonizBj39yGwUzVDVTlMYJMK2Wp6xdpy72mfArFq5uK+NOuexfzZlzI1z9+AQ== + dependencies: + "@babel/helper-get-function-arity" "^7.14.5" + "@babel/template" "^7.14.5" + "@babel/types" "^7.14.5" + +"@babel/helper-get-function-arity@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.14.5.tgz#25fbfa579b0937eee1f3b805ece4ce398c431815" + integrity sha512-I1Db4Shst5lewOM4V+ZKJzQ0JGGaZ6VY1jYvMghRjqs6DWgxLCIyFt30GlnKkfUeFLpJt2vzbMVEXVSXlIFYUg== + dependencies: + "@babel/types" "^7.14.5" + +"@babel/helper-hoist-variables@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.14.5.tgz#e0dd27c33a78e577d7c8884916a3e7ef1f7c7f8d" + integrity sha512-R1PXiz31Uc0Vxy4OEOm07x0oSjKAdPPCh3tPivn/Eo8cvz6gveAeuyUUPB21Hoiif0uoPQSSdhIPS3352nvdyQ== + dependencies: + "@babel/types" "^7.14.5" + +"@babel/helper-member-expression-to-functions@^7.14.5": + version "7.14.7" + resolved "https://registry.yarnpkg.com/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.14.7.tgz#97e56244beb94211fe277bd818e3a329c66f7970" + integrity sha512-TMUt4xKxJn6ccjcOW7c4hlwyJArizskAhoSTOCkA0uZ+KghIaci0Qg9R043kUMWI9mtQfgny+NQ5QATnZ+paaA== + dependencies: + "@babel/types" "^7.14.5" + +"@babel/helper-module-imports@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.14.5.tgz#6d1a44df6a38c957aa7c312da076429f11b422f3" + integrity sha512-SwrNHu5QWS84XlHwGYPDtCxcA0hrSlL2yhWYLgeOc0w7ccOl2qv4s/nARI0aYZW+bSwAL5CukeXA47B/1NKcnQ== + dependencies: + "@babel/types" "^7.14.5" + +"@babel/helper-module-transforms@^7.14.8": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.14.8.tgz#d4279f7e3fd5f4d5d342d833af36d4dd87d7dc49" + integrity sha512-RyE+NFOjXn5A9YU1dkpeBaduagTlZ0+fccnIcAGbv1KGUlReBj7utF7oEth8IdIBQPcux0DDgW5MFBH2xu9KcA== + dependencies: + "@babel/helper-module-imports" "^7.14.5" + "@babel/helper-replace-supers" "^7.14.5" + "@babel/helper-simple-access" "^7.14.8" + "@babel/helper-split-export-declaration" "^7.14.5" + "@babel/helper-validator-identifier" "^7.14.8" + "@babel/template" "^7.14.5" + "@babel/traverse" "^7.14.8" + "@babel/types" "^7.14.8" + +"@babel/helper-optimise-call-expression@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.14.5.tgz#f27395a8619e0665b3f0364cddb41c25d71b499c" + integrity sha512-IqiLIrODUOdnPU9/F8ib1Fx2ohlgDhxnIDU7OEVi+kAbEZcyiF7BLU8W6PfvPi9LzztjS7kcbzbmL7oG8kD6VA== + dependencies: + "@babel/types" "^7.14.5" + +"@babel/helper-plugin-utils@^7.0.0", "@babel/helper-plugin-utils@^7.10.4", "@babel/helper-plugin-utils@^7.12.13", "@babel/helper-plugin-utils@^7.14.5", "@babel/helper-plugin-utils@^7.8.0": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-plugin-utils/-/helper-plugin-utils-7.14.5.tgz#5ac822ce97eec46741ab70a517971e443a70c5a9" + integrity sha512-/37qQCE3K0vvZKwoK4XU/irIJQdIfCJuhU5eKnNxpFDsOkgFaUAwbv+RYw6eYgsC0E4hS7r5KqGULUogqui0fQ== + +"@babel/helper-replace-supers@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-replace-supers/-/helper-replace-supers-7.14.5.tgz#0ecc0b03c41cd567b4024ea016134c28414abb94" + integrity sha512-3i1Qe9/8x/hCHINujn+iuHy+mMRLoc77b2nI9TB0zjH1hvn9qGlXjWlggdwUcju36PkPCy/lpM7LLUdcTyH4Ow== + dependencies: + "@babel/helper-member-expression-to-functions" "^7.14.5" + "@babel/helper-optimise-call-expression" "^7.14.5" + "@babel/traverse" "^7.14.5" + "@babel/types" "^7.14.5" + +"@babel/helper-simple-access@^7.14.8": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/helper-simple-access/-/helper-simple-access-7.14.8.tgz#82e1fec0644a7e775c74d305f212c39f8fe73924" + integrity sha512-TrFN4RHh9gnWEU+s7JloIho2T76GPwRHhdzOWLqTrMnlas8T9O7ec+oEDNsRXndOmru9ymH9DFrEOxpzPoSbdg== + dependencies: + "@babel/types" "^7.14.8" + +"@babel/helper-split-export-declaration@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.14.5.tgz#22b23a54ef51c2b7605d851930c1976dd0bc693a" + integrity sha512-hprxVPu6e5Kdp2puZUmvOGjaLv9TCe58E/Fl6hRq4YiVQxIcNvuq6uTM2r1mT/oPskuS9CgR+I94sqAYv0NGKA== + dependencies: + "@babel/types" "^7.14.5" + +"@babel/helper-validator-identifier@^7.14.5", "@babel/helper-validator-identifier@^7.14.8": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.14.8.tgz#32be33a756f29e278a0d644fa08a2c9e0f88a34c" + integrity sha512-ZGy6/XQjllhYQrNw/3zfWRwZCTVSiBLZ9DHVZxn9n2gip/7ab8mv2TWlKPIBk26RwedCBoWdjLmn+t9na2Gcow== + +"@babel/helper-validator-option@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-option/-/helper-validator-option-7.14.5.tgz#6e72a1fff18d5dfcb878e1e62f1a021c4b72d5a3" + integrity sha512-OX8D5eeX4XwcroVW45NMvoYaIuFI+GQpA2a8Gi+X/U/cDUIRsV37qQfF905F0htTRCREQIB4KqPeaveRJUl3Ow== + +"@babel/helpers@^7.14.8": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.14.8.tgz#839f88f463025886cff7f85a35297007e2da1b77" + integrity sha512-ZRDmI56pnV+p1dH6d+UN6GINGz7Krps3+270qqI9UJ4wxYThfAIcI5i7j5vXC4FJ3Wap+S9qcebxeYiqn87DZw== + dependencies: + "@babel/template" "^7.14.5" + "@babel/traverse" "^7.14.8" + "@babel/types" "^7.14.8" + +"@babel/highlight@^7.10.4", "@babel/highlight@^7.14.5": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.14.5.tgz#6861a52f03966405001f6aa534a01a24d99e8cd9" + integrity sha512-qf9u2WFWVV0MppaL877j2dBtQIDgmidgjGk5VIMw3OadXvYaXn66U1BFlH2t4+t3i+8PhedppRv+i40ABzd+gg== + dependencies: + "@babel/helper-validator-identifier" "^7.14.5" + chalk "^2.0.0" + js-tokens "^4.0.0" + +"@babel/parser@^7.1.0", "@babel/parser@^7.14.5", "@babel/parser@^7.14.8", "@babel/parser@^7.7.2": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.14.8.tgz#66fd41666b2d7b840bd5ace7f7416d5ac60208d4" + integrity sha512-syoCQFOoo/fzkWDeM0dLEZi5xqurb5vuyzwIMNZRNun+N/9A4cUZeQaE7dTrB8jGaKuJRBtEOajtnmw0I5hvvA== + +"@babel/plugin-syntax-async-generators@^7.8.4": + version "7.8.4" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz#a983fb1aeb2ec3f6ed042a210f640e90e786fe0d" + integrity sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-bigint@^7.8.3": + version "7.8.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz#4c9a6f669f5d0cdf1b90a1671e9a146be5300cea" + integrity sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-class-properties@^7.8.3": + version "7.12.13" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz#b5c987274c4a3a82b89714796931a6b53544ae10" + integrity sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA== + dependencies: + "@babel/helper-plugin-utils" "^7.12.13" + +"@babel/plugin-syntax-import-meta@^7.8.3": + version "7.10.4" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz#ee601348c370fa334d2207be158777496521fd51" + integrity sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g== + dependencies: + "@babel/helper-plugin-utils" "^7.10.4" + +"@babel/plugin-syntax-json-strings@^7.8.3": + version "7.8.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz#01ca21b668cd8218c9e640cb6dd88c5412b2c96a" + integrity sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-logical-assignment-operators@^7.8.3": + version "7.10.4" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz#ca91ef46303530448b906652bac2e9fe9941f699" + integrity sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig== + dependencies: + "@babel/helper-plugin-utils" "^7.10.4" + +"@babel/plugin-syntax-nullish-coalescing-operator@^7.8.3": + version "7.8.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz#167ed70368886081f74b5c36c65a88c03b66d1a9" + integrity sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-numeric-separator@^7.8.3": + version "7.10.4" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz#b9b070b3e33570cd9fd07ba7fa91c0dd37b9af97" + integrity sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug== + dependencies: + "@babel/helper-plugin-utils" "^7.10.4" + +"@babel/plugin-syntax-object-rest-spread@^7.8.3": + version "7.8.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz#60e225edcbd98a640332a2e72dd3e66f1af55871" + integrity sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-optional-catch-binding@^7.8.3": + version "7.8.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz#6111a265bcfb020eb9efd0fdfd7d26402b9ed6c1" + integrity sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-optional-chaining@^7.8.3": + version "7.8.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz#4f69c2ab95167e0180cd5336613f8c5788f7d48a" + integrity sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg== + dependencies: + "@babel/helper-plugin-utils" "^7.8.0" + +"@babel/plugin-syntax-top-level-await@^7.8.3": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz#c1cfdadc35a646240001f06138247b741c34d94c" + integrity sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw== + dependencies: + "@babel/helper-plugin-utils" "^7.14.5" + +"@babel/plugin-syntax-typescript@^7.7.2": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.14.5.tgz#b82c6ce471b165b5ce420cf92914d6fb46225716" + integrity sha512-u6OXzDaIXjEstBRRoBCQ/uKQKlbuaeE5in0RvWdA4pN6AhqxTIwUsnHPU1CFZA/amYObMsuWhYfRl3Ch90HD0Q== + dependencies: + "@babel/helper-plugin-utils" "^7.14.5" + +"@babel/template@^7.14.5", "@babel/template@^7.3.3": + version "7.14.5" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.14.5.tgz#a9bc9d8b33354ff6e55a9c60d1109200a68974f4" + integrity sha512-6Z3Po85sfxRGachLULUhOmvAaOo7xCvqGQtxINai2mEGPFm6pQ4z5QInFnUrRpfoSV60BnjyF5F3c+15fxFV1g== + dependencies: + "@babel/code-frame" "^7.14.5" + "@babel/parser" "^7.14.5" + "@babel/types" "^7.14.5" + +"@babel/traverse@^7.1.0", "@babel/traverse@^7.14.5", "@babel/traverse@^7.14.8", "@babel/traverse@^7.7.2": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.14.8.tgz#c0253f02677c5de1a8ff9df6b0aacbec7da1a8ce" + integrity sha512-kexHhzCljJcFNn1KYAQ6A5wxMRzq9ebYpEDV4+WdNyr3i7O44tanbDOR/xjiG2F3sllan+LgwK+7OMk0EmydHg== + dependencies: + "@babel/code-frame" "^7.14.5" + "@babel/generator" "^7.14.8" + "@babel/helper-function-name" "^7.14.5" + "@babel/helper-hoist-variables" "^7.14.5" + "@babel/helper-split-export-declaration" "^7.14.5" + "@babel/parser" "^7.14.8" + "@babel/types" "^7.14.8" + debug "^4.1.0" + globals "^11.1.0" + +"@babel/types@^7.0.0", "@babel/types@^7.14.5", "@babel/types@^7.14.8", "@babel/types@^7.3.0", "@babel/types@^7.3.3": + version "7.14.8" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.14.8.tgz#38109de8fcadc06415fbd9b74df0065d4d41c728" + integrity sha512-iob4soQa7dZw8nodR/KlOQkPh9S4I8RwCxwRIFuiMRYjOzH/KJzdUfDgz6cGi5dDaclXF4P2PAhCdrBJNIg68Q== + dependencies: + "@babel/helper-validator-identifier" "^7.14.8" + to-fast-properties "^2.0.0" + +"@bcoe/v8-coverage@^0.2.3": + version "0.2.3" + resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" + integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== + +"@eslint/eslintrc@^0.4.3": + version "0.4.3" + resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-0.4.3.tgz#9e42981ef035beb3dd49add17acb96e8ff6f394c" + integrity sha512-J6KFFz5QCYUJq3pf0mjEcCJVERbzv71PUIDczuh9JkwGEzced6CO5ADLHB1rbf/+oPBtoPfMYNOpGDzCANlbXw== + dependencies: + ajv "^6.12.4" + debug "^4.1.1" + espree "^7.3.0" + globals "^13.9.0" + ignore "^4.0.6" + import-fresh "^3.2.1" + js-yaml "^3.13.1" + minimatch "^3.0.4" + strip-json-comments "^3.1.1" + +"@gulp-sourcemaps/identity-map@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@gulp-sourcemaps/identity-map/-/identity-map-2.0.1.tgz#a6e8b1abec8f790ec6be2b8c500e6e68037c0019" + integrity sha512-Tb+nSISZku+eQ4X1lAkevcQa+jknn/OVUgZ3XCxEKIsLsqYuPoJwJOPQeaOk75X3WPftb29GWY1eqE7GLsXb1Q== + dependencies: + acorn "^6.4.1" + normalize-path "^3.0.0" + postcss "^7.0.16" + source-map "^0.6.0" + through2 "^3.0.1" + +"@gulp-sourcemaps/map-sources@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@gulp-sourcemaps/map-sources/-/map-sources-1.0.0.tgz#890ae7c5d8c877f6d384860215ace9d7ec945bda" + integrity sha1-iQrnxdjId/bThIYCFazp1+yUW9o= + dependencies: + normalize-path "^2.0.1" + through2 "^2.0.3" + +"@humanwhocodes/config-array@^0.5.0": + version "0.5.0" + resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.5.0.tgz#1407967d4c6eecd7388f83acf1eaf4d0c6e58ef9" + integrity sha512-FagtKFz74XrTl7y6HCzQpwDfXP0yhxe9lHLD1UZxjvZIcbyRz8zTFF/yYNfSfzU414eDwZ1SrO0Qvtyf+wFMQg== + dependencies: + "@humanwhocodes/object-schema" "^1.2.0" + debug "^4.1.1" + minimatch "^3.0.4" + +"@humanwhocodes/object-schema@^1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.0.tgz#87de7af9c231826fdd68ac7258f77c429e0e5fcf" + integrity sha512-wdppn25U8z/2yiaT6YGquE6X8sSv7hNMWSXYSSU1jGv/yd6XqjXgTDJ8KP4NgjTXfJ3GbRjeeb8RTV7a/VpM+w== + +"@hutson/parse-repository-url@^3.0.0": + version "3.0.2" + resolved "https://registry.yarnpkg.com/@hutson/parse-repository-url/-/parse-repository-url-3.0.2.tgz#98c23c950a3d9b6c8f0daed06da6c3af06981340" + integrity sha512-H9XAx3hc0BQHY6l+IFSWHDySypcXsvsuLhgYLUGywmJ5pswRVQJUHpOsobnLYp2ZUaUlKiKDrgWWhosOwAEM8Q== + +"@istanbuljs/load-nyc-config@^1.0.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz#fd3db1d59ecf7cf121e80650bb86712f9b55eced" + integrity sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ== + dependencies: + camelcase "^5.3.1" + find-up "^4.1.0" + get-package-type "^0.1.0" + js-yaml "^3.13.1" + resolve-from "^5.0.0" + +"@istanbuljs/schema@^0.1.2": + version "0.1.3" + resolved "https://registry.yarnpkg.com/@istanbuljs/schema/-/schema-0.1.3.tgz#e45e384e4b8ec16bce2fd903af78450f6bf7ec98" + integrity sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA== + +"@jest/console@^27.0.6": + version "27.0.6" + resolved "https://registry.yarnpkg.com/@jest/console/-/console-27.0.6.tgz#3eb72ea80897495c3d73dd97aab7f26770e2260f" + integrity sha512-fMlIBocSHPZ3JxgWiDNW/KPj6s+YRd0hicb33IrmelCcjXo/pXPwvuiKFmZz+XuqI/1u7nbUK10zSsWL/1aegg== + dependencies: + "@jest/types" "^27.0.6" + "@types/node" "*" + chalk "^4.0.0" + jest-message-util "^27.0.6" + jest-util "^27.0.6" + slash "^3.0.0" + +"@jest/core@^27.0.6": + version "27.0.6" + resolved "https://registry.yarnpkg.com/@jest/core/-/core-27.0.6.tgz#c5f642727a0b3bf0f37c4b46c675372d0978d4a1" + integrity sha512-SsYBm3yhqOn5ZLJCtccaBcvD/ccTLCeuDv8U41WJH/V1MW5eKUkeMHT9U+Pw/v1m1AIWlnIW/eM2XzQr0rEmow== + dependencies: + "@jest/console" "^27.0.6" + "@jest/reporters" "^27.0.6" + "@jest/test-result" "^27.0.6" + "@jest/transform" "^27.0.6" + "@jest/types" "^27.0.6" + "@types/node" "*" + ansi-escapes "^4.2.1" + chalk "^4.0.0" + emittery "^0.8.1" + exit "^0.1.2" + graceful-fs "^4.2.4" + jest-changed-files "^27.0.6" + jest-config "^27.0.6" + jest-haste-map "^27.0.6" + jest-message-util "^27.0.6" + jest-regex-util "^27.0.6" + jest-resolve "^27.0.6" + jest-resolve-dependencies "^27.0.6" + jest-runner "^27.0.6" + jest-runtime "^27.0.6" + jest-snapshot "^27.0.6" + jest-util "^27.0.6" + jest-validate "^27.0.6" + jest-watcher "^27.0.6" + micromatch "^4.0.4" + p-each-series "^2.1.0" + rimraf "^3.0.0" + slash "^3.0.0" + strip-ansi "^6.0.0" + +"@jest/environment@^27.0.6": + version "27.0.6" + resolved "https://registry.yarnpkg.com/@jest/environment/-/environment-27.0.6.tgz#ee293fe996db01d7d663b8108fa0e1ff436219d2" + integrity sha512-4XywtdhwZwCpPJ/qfAkqExRsERW+UaoSRStSHCCiQTUpoYdLukj+YJbQSFrZjhlUDRZeNiU9SFH0u7iNimdiIg== + dependencies: + "@jest/fake-timers" "^27.0.6" + "@jest/types" "^27.0.6" + "@types/node" "*" + jest-mock "^27.0.6" + +"@jest/fake-timers@^27.0.6": + version "27.0.6" + resolved "https://registry.yarnpkg.com/@jest/fake-timers/-/fake-timers-27.0.6.tgz#cbad52f3fe6abe30e7acb8cd5fa3466b9588e3df" + integrity sha512-sqd+xTWtZ94l3yWDKnRTdvTeZ+A/V7SSKrxsrOKSqdyddb9CeNRF8fbhAU0D7ZJBpTTW2nbp6MftmKJDZfW2LQ== + dependencies: + "@jest/types" "^27.0.6" + "@sinonjs/fake-timers" "^7.0.2" + "@types/node" "*" + jest-message-util "^27.0.6" + jest-mock "^27.0.6" + jest-util "^27.0.6" + +"@jest/globals@^27.0.6": + version "27.0.6" + resolved "https://registry.yarnpkg.com/@jest/globals/-/globals-27.0.6.tgz#48e3903f99a4650673d8657334d13c9caf0e8f82" + integrity sha512-DdTGCP606rh9bjkdQ7VvChV18iS7q0IMJVP1piwTWyWskol4iqcVwthZmoJEf7obE1nc34OpIyoVGPeqLC+ryw== + dependencies: + "@jest/environment" "^27.0.6" + "@jest/types" "^27.0.6" + expect "^27.0.6" + +"@jest/reporters@^27.0.6": + version "27.0.6" + resolved "https://registry.yarnpkg.com/@jest/reporters/-/reporters-27.0.6.tgz#91e7f2d98c002ad5df94d5b5167c1eb0b9fd5b00" + integrity sha512-TIkBt09Cb2gptji3yJXb3EE+eVltW6BjO7frO7NEfjI9vSIYoISi5R3aI3KpEDXlB1xwB+97NXIqz84qYeYsfA== + dependencies: + "@bcoe/v8-coverage" "^0.2.3" + "@jest/console" "^27.0.6" + "@jest/test-result" "^27.0.6" + "@jest/transform" "^27.0.6" + "@jest/types" "^27.0.6" + chalk "^4.0.0" + collect-v8-coverage "^1.0.0" + exit "^0.1.2" + glob "^7.1.2" + graceful-fs "^4.2.4" + istanbul-lib-coverage "^3.0.0" + istanbul-lib-instrument "^4.0.3" + istanbul-lib-report "^3.0.0" + istanbul-lib-source-maps "^4.0.0" + istanbul-reports "^3.0.2" + jest-haste-map "^27.0.6" + jest-resolve "^27.0.6" + jest-util "^27.0.6" + jest-worker "^27.0.6" + slash "^3.0.0" + source-map "^0.6.0" + string-length "^4.0.1" + terminal-link "^2.0.0" + v8-to-istanbul "^8.0.0" + +"@jest/source-map@^27.0.6": + version "27.0.6" + resolved "https://registry.yarnpkg.com/@jest/source-map/-/source-map-27.0.6.tgz#be9e9b93565d49b0548b86e232092491fb60551f" + integrity sha512-Fek4mi5KQrqmlY07T23JRi0e7Z9bXTOOD86V/uS0EIW4PClvPDqZOyFlLpNJheS6QI0FNX1CgmPjtJ4EA/2M+g== + dependencies: + callsites "^3.0.0" + graceful-fs "^4.2.4" + source-map "^0.6.0" + +"@jest/test-result@^27.0.6": + version "27.0.6" + resolved "https://registry.yarnpkg.com/@jest/test-result/-/test-result-27.0.6.tgz#3fa42015a14e4fdede6acd042ce98c7f36627051" + integrity sha512-ja/pBOMTufjX4JLEauLxE3LQBPaI2YjGFtXexRAjt1I/MbfNlMx0sytSX3tn5hSLzQsR3Qy2rd0hc1BWojtj9w== + dependencies: + "@jest/console" "^27.0.6" + "@jest/types" "^27.0.6" + "@types/istanbul-lib-coverage" "^2.0.0" + collect-v8-coverage "^1.0.0" + +"@jest/test-sequencer@^27.0.6": + version "27.0.6" + resolved "https://registry.yarnpkg.com/@jest/test-sequencer/-/test-sequencer-27.0.6.tgz#80a913ed7a1130545b1cd777ff2735dd3af5d34b" + integrity sha512-bISzNIApazYOlTHDum9PwW22NOyDa6VI31n6JucpjTVM0jD6JDgqEZ9+yn575nDdPF0+4csYDxNNW13NvFQGZA== + dependencies: + "@jest/test-result" "^27.0.6" + graceful-fs "^4.2.4" + jest-haste-map "^27.0.6" + jest-runtime "^27.0.6" + +"@jest/transform@^27.0.6": + version "27.0.6" + resolved "https://registry.yarnpkg.com/@jest/transform/-/transform-27.0.6.tgz#189ad7107413208f7600f4719f81dd2f7278cc95" + integrity sha512-rj5Dw+mtIcntAUnMlW/Vju5mr73u8yg+irnHwzgtgoeI6cCPOvUwQ0D1uQtc/APmWgvRweEb1g05pkUpxH3iCA== + dependencies: + "@babel/core" "^7.1.0" + "@jest/types" "^27.0.6" + babel-plugin-istanbul "^6.0.0" + chalk "^4.0.0" + convert-source-map "^1.4.0" + fast-json-stable-stringify "^2.0.0" + graceful-fs "^4.2.4" + jest-haste-map "^27.0.6" + jest-regex-util "^27.0.6" + jest-util "^27.0.6" + micromatch "^4.0.4" + pirates "^4.0.1" + slash "^3.0.0" + source-map "^0.6.1" + write-file-atomic "^3.0.0" + +"@jest/types@^26.6.2": + version "26.6.2" + resolved "https://registry.yarnpkg.com/@jest/types/-/types-26.6.2.tgz#bef5a532030e1d88a2f5a6d933f84e97226ed48e" + integrity sha512-fC6QCp7Sc5sX6g8Tvbmj4XUTbyrik0akgRy03yjXbQaBWWNWGE7SGtJk98m0N8nzegD/7SggrUlivxo5ax4KWQ== + dependencies: + "@types/istanbul-lib-coverage" "^2.0.0" + "@types/istanbul-reports" "^3.0.0" + "@types/node" "*" + "@types/yargs" "^15.0.0" + chalk "^4.0.0" + +"@jest/types@^27.0.6": + version "27.0.6" + resolved "https://registry.yarnpkg.com/@jest/types/-/types-27.0.6.tgz#9a992bc517e0c49f035938b8549719c2de40706b" + integrity sha512-aSquT1qa9Pik26JK5/3rvnYb4bGtm1VFNesHKmNTwmPIgOrixvhL2ghIvFRNEpzy3gU+rUgjIF/KodbkFAl++g== + dependencies: + "@types/istanbul-lib-coverage" "^2.0.0" + "@types/istanbul-reports" "^3.0.0" + "@types/node" "*" + "@types/yargs" "^16.0.0" + chalk "^4.0.0" + +"@lerna/add@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/add/-/add-4.0.0.tgz#c36f57d132502a57b9e7058d1548b7a565ef183f" + integrity sha512-cpmAH1iS3k8JBxNvnMqrGTTjbY/ZAiKa1ChJzFevMYY3eeqbvhsBKnBcxjRXtdrJ6bd3dCQM+ZtK+0i682Fhng== + dependencies: + "@lerna/bootstrap" "4.0.0" + "@lerna/command" "4.0.0" + "@lerna/filter-options" "4.0.0" + "@lerna/npm-conf" "4.0.0" + "@lerna/validation-error" "4.0.0" + dedent "^0.7.0" + npm-package-arg "^8.1.0" + p-map "^4.0.0" + pacote "^11.2.6" + semver "^7.3.4" + +"@lerna/bootstrap@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/bootstrap/-/bootstrap-4.0.0.tgz#5f5c5e2c6cfc8fcec50cb2fbe569a8c607101891" + integrity sha512-RkS7UbeM2vu+kJnHzxNRCLvoOP9yGNgkzRdy4UV2hNalD7EP41bLvRVOwRYQ7fhc2QcbhnKNdOBihYRL0LcKtw== + dependencies: + "@lerna/command" "4.0.0" + "@lerna/filter-options" "4.0.0" + "@lerna/has-npm-version" "4.0.0" + "@lerna/npm-install" "4.0.0" + "@lerna/package-graph" "4.0.0" + "@lerna/pulse-till-done" "4.0.0" + "@lerna/rimraf-dir" "4.0.0" + "@lerna/run-lifecycle" "4.0.0" + "@lerna/run-topologically" "4.0.0" + "@lerna/symlink-binary" "4.0.0" + "@lerna/symlink-dependencies" "4.0.0" + "@lerna/validation-error" "4.0.0" + dedent "^0.7.0" + get-port "^5.1.1" + multimatch "^5.0.0" + npm-package-arg "^8.1.0" + npmlog "^4.1.2" + p-map "^4.0.0" + p-map-series "^2.1.0" + p-waterfall "^2.1.1" + read-package-tree "^5.3.1" + semver "^7.3.4" + +"@lerna/changed@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/changed/-/changed-4.0.0.tgz#b9fc76cea39b9292a6cd263f03eb57af85c9270b" + integrity sha512-cD+KuPRp6qiPOD+BO6S6SN5cARspIaWSOqGBpGnYzLb4uWT8Vk4JzKyYtc8ym1DIwyoFXHosXt8+GDAgR8QrgQ== + dependencies: + "@lerna/collect-updates" "4.0.0" + "@lerna/command" "4.0.0" + "@lerna/listable" "4.0.0" + "@lerna/output" "4.0.0" + +"@lerna/check-working-tree@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/check-working-tree/-/check-working-tree-4.0.0.tgz#257e36a602c00142e76082a19358e3e1ae8dbd58" + integrity sha512-/++bxM43jYJCshBiKP5cRlCTwSJdRSxVmcDAXM+1oUewlZJVSVlnks5eO0uLxokVFvLhHlC5kHMc7gbVFPHv6Q== + dependencies: + "@lerna/collect-uncommitted" "4.0.0" + "@lerna/describe-ref" "4.0.0" + "@lerna/validation-error" "4.0.0" + +"@lerna/child-process@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/child-process/-/child-process-4.0.0.tgz#341b96a57dffbd9705646d316e231df6fa4df6e1" + integrity sha512-XtCnmCT9eyVsUUHx6y/CTBYdV9g2Cr/VxyseTWBgfIur92/YKClfEtJTbOh94jRT62hlKLqSvux/UhxXVh613Q== + dependencies: + chalk "^4.1.0" + execa "^5.0.0" + strong-log-transformer "^2.1.0" + +"@lerna/clean@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/clean/-/clean-4.0.0.tgz#8f778b6f2617aa2a936a6b5e085ae62498e57dc5" + integrity sha512-uugG2iN9k45ITx2jtd8nEOoAtca8hNlDCUM0N3lFgU/b1mEQYAPRkqr1qs4FLRl/Y50ZJ41wUz1eazS+d/0osA== + dependencies: + "@lerna/command" "4.0.0" + "@lerna/filter-options" "4.0.0" + "@lerna/prompt" "4.0.0" + "@lerna/pulse-till-done" "4.0.0" + "@lerna/rimraf-dir" "4.0.0" + p-map "^4.0.0" + p-map-series "^2.1.0" + p-waterfall "^2.1.1" + +"@lerna/cli@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/cli/-/cli-4.0.0.tgz#8eabd334558836c1664df23f19acb95e98b5bbf3" + integrity sha512-Neaw3GzFrwZiRZv2g7g6NwFjs3er1vhraIniEs0jjVLPMNC4eata0na3GfE5yibkM/9d3gZdmihhZdZ3EBdvYA== + dependencies: + "@lerna/global-options" "4.0.0" + dedent "^0.7.0" + npmlog "^4.1.2" + yargs "^16.2.0" + +"@lerna/collect-uncommitted@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/collect-uncommitted/-/collect-uncommitted-4.0.0.tgz#855cd64612969371cfc2453b90593053ff1ba779" + integrity sha512-ufSTfHZzbx69YNj7KXQ3o66V4RC76ffOjwLX0q/ab//61bObJ41n03SiQEhSlmpP+gmFbTJ3/7pTe04AHX9m/g== + dependencies: + "@lerna/child-process" "4.0.0" + chalk "^4.1.0" + npmlog "^4.1.2" + +"@lerna/collect-updates@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/collect-updates/-/collect-updates-4.0.0.tgz#8e208b1bafd98a372ff1177f7a5e288f6bea8041" + integrity sha512-bnNGpaj4zuxsEkyaCZLka9s7nMs58uZoxrRIPJ+nrmrZYp1V5rrd+7/NYTuunOhY2ug1sTBvTAxj3NZQ+JKnOw== + dependencies: + "@lerna/child-process" "4.0.0" + "@lerna/describe-ref" "4.0.0" + minimatch "^3.0.4" + npmlog "^4.1.2" + slash "^3.0.0" + +"@lerna/command@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/command/-/command-4.0.0.tgz#991c7971df8f5bf6ae6e42c808869a55361c1b98" + integrity sha512-LM9g3rt5FsPNFqIHUeRwWXLNHJ5NKzOwmVKZ8anSp4e1SPrv2HNc1V02/9QyDDZK/w+5POXH5lxZUI1CHaOK/A== + dependencies: + "@lerna/child-process" "4.0.0" + "@lerna/package-graph" "4.0.0" + "@lerna/project" "4.0.0" + "@lerna/validation-error" "4.0.0" + "@lerna/write-log-file" "4.0.0" + clone-deep "^4.0.1" + dedent "^0.7.0" + execa "^5.0.0" + is-ci "^2.0.0" + npmlog "^4.1.2" + +"@lerna/conventional-commits@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/conventional-commits/-/conventional-commits-4.0.0.tgz#660fb2c7b718cb942ead70110df61f18c6f99750" + integrity sha512-CSUQRjJHFrH8eBn7+wegZLV3OrNc0Y1FehYfYGhjLE2SIfpCL4bmfu/ViYuHh9YjwHaA+4SX6d3hR+xkeseKmw== + dependencies: + "@lerna/validation-error" "4.0.0" + conventional-changelog-angular "^5.0.12" + conventional-changelog-core "^4.2.2" + conventional-recommended-bump "^6.1.0" + fs-extra "^9.1.0" + get-stream "^6.0.0" + lodash.template "^4.5.0" + npm-package-arg "^8.1.0" + npmlog "^4.1.2" + pify "^5.0.0" + semver "^7.3.4" + +"@lerna/create-symlink@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/create-symlink/-/create-symlink-4.0.0.tgz#8c5317ce5ae89f67825443bd7651bf4121786228" + integrity sha512-I0phtKJJdafUiDwm7BBlEUOtogmu8+taxq6PtIrxZbllV9hWg59qkpuIsiFp+no7nfRVuaasNYHwNUhDAVQBig== + dependencies: + cmd-shim "^4.1.0" + fs-extra "^9.1.0" + npmlog "^4.1.2" + +"@lerna/create@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/create/-/create-4.0.0.tgz#b6947e9b5dfb6530321952998948c3e63d64d730" + integrity sha512-mVOB1niKByEUfxlbKTM1UNECWAjwUdiioIbRQZEeEabtjCL69r9rscIsjlGyhGWCfsdAG5wfq4t47nlDXdLLag== + dependencies: + "@lerna/child-process" "4.0.0" + "@lerna/command" "4.0.0" + "@lerna/npm-conf" "4.0.0" + "@lerna/validation-error" "4.0.0" + dedent "^0.7.0" + fs-extra "^9.1.0" + globby "^11.0.2" + init-package-json "^2.0.2" + npm-package-arg "^8.1.0" + p-reduce "^2.1.0" + pacote "^11.2.6" + pify "^5.0.0" + semver "^7.3.4" + slash "^3.0.0" + validate-npm-package-license "^3.0.4" + validate-npm-package-name "^3.0.0" + whatwg-url "^8.4.0" + yargs-parser "20.2.4" + +"@lerna/describe-ref@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/describe-ref/-/describe-ref-4.0.0.tgz#53c53b4ea65fdceffa072a62bfebe6772c45d9ec" + integrity sha512-eTU5+xC4C5Gcgz+Ey4Qiw9nV2B4JJbMulsYJMW8QjGcGh8zudib7Sduj6urgZXUYNyhYpRs+teci9M2J8u+UvQ== + dependencies: + "@lerna/child-process" "4.0.0" + npmlog "^4.1.2" + +"@lerna/diff@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/diff/-/diff-4.0.0.tgz#6d3071817aaa4205a07bf77cfc6e932796d48b92" + integrity sha512-jYPKprQVg41+MUMxx6cwtqsNm0Yxx9GDEwdiPLwcUTFx+/qKCEwifKNJ1oGIPBxyEHX2PFCOjkK39lHoj2qiag== + dependencies: + "@lerna/child-process" "4.0.0" + "@lerna/command" "4.0.0" + "@lerna/validation-error" "4.0.0" + npmlog "^4.1.2" + +"@lerna/exec@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/exec/-/exec-4.0.0.tgz#eb6cb95cb92d42590e9e2d628fcaf4719d4a8be6" + integrity sha512-VGXtL/b/JfY84NB98VWZpIExfhLOzy0ozm/0XaS4a2SmkAJc5CeUfrhvHxxkxiTBLkU+iVQUyYEoAT0ulQ8PCw== + dependencies: + "@lerna/child-process" "4.0.0" + "@lerna/command" "4.0.0" + "@lerna/filter-options" "4.0.0" + "@lerna/profiler" "4.0.0" + "@lerna/run-topologically" "4.0.0" + "@lerna/validation-error" "4.0.0" + p-map "^4.0.0" + +"@lerna/filter-options@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/filter-options/-/filter-options-4.0.0.tgz#ac94cc515d7fa3b47e2f7d74deddeabb1de5e9e6" + integrity sha512-vV2ANOeZhOqM0rzXnYcFFCJ/kBWy/3OA58irXih9AMTAlQLymWAK0akWybl++sUJ4HB9Hx12TOqaXbYS2NM5uw== + dependencies: + "@lerna/collect-updates" "4.0.0" + "@lerna/filter-packages" "4.0.0" + dedent "^0.7.0" + npmlog "^4.1.2" + +"@lerna/filter-packages@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/filter-packages/-/filter-packages-4.0.0.tgz#b1f70d70e1de9cdd36a4e50caa0ac501f8d012f2" + integrity sha512-+4AJIkK7iIiOaqCiVTYJxh/I9qikk4XjNQLhE3kixaqgMuHl1NQ99qXRR0OZqAWB9mh8Z1HA9bM5K1HZLBTOqA== + dependencies: + "@lerna/validation-error" "4.0.0" + multimatch "^5.0.0" + npmlog "^4.1.2" + +"@lerna/get-npm-exec-opts@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/get-npm-exec-opts/-/get-npm-exec-opts-4.0.0.tgz#dc955be94a4ae75c374ef9bce91320887d34608f" + integrity sha512-yvmkerU31CTWS2c7DvmAWmZVeclPBqI7gPVr5VATUKNWJ/zmVcU4PqbYoLu92I9Qc4gY1TuUplMNdNuZTSL7IQ== + dependencies: + npmlog "^4.1.2" + +"@lerna/get-packed@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/get-packed/-/get-packed-4.0.0.tgz#0989d61624ac1f97e393bdad2137c49cd7a37823" + integrity sha512-rfWONRsEIGyPJTxFzC8ECb3ZbsDXJbfqWYyeeQQDrJRPnEJErlltRLPLgC2QWbxFgFPsoDLeQmFHJnf0iDfd8w== + dependencies: + fs-extra "^9.1.0" + ssri "^8.0.1" + tar "^6.1.0" + +"@lerna/github-client@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/github-client/-/github-client-4.0.0.tgz#2ced67721363ef70f8e12ffafce4410918f4a8a4" + integrity sha512-2jhsldZtTKXYUBnOm23Lb0Fx8G4qfSXF9y7UpyUgWUj+YZYd+cFxSuorwQIgk5P4XXrtVhsUesIsli+BYSThiw== + dependencies: + "@lerna/child-process" "4.0.0" + "@octokit/plugin-enterprise-rest" "^6.0.1" + "@octokit/rest" "^18.1.0" + git-url-parse "^11.4.4" + npmlog "^4.1.2" + +"@lerna/gitlab-client@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/gitlab-client/-/gitlab-client-4.0.0.tgz#00dad73379c7b38951d4b4ded043504c14e2b67d" + integrity sha512-OMUpGSkeDWFf7BxGHlkbb35T7YHqVFCwBPSIR6wRsszY8PAzCYahtH3IaJzEJyUg6vmZsNl0FSr3pdA2skhxqA== + dependencies: + node-fetch "^2.6.1" + npmlog "^4.1.2" + whatwg-url "^8.4.0" + +"@lerna/global-options@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/global-options/-/global-options-4.0.0.tgz#c7d8b0de6a01d8a845e2621ea89e7f60f18c6a5f" + integrity sha512-TRMR8afAHxuYBHK7F++Ogop2a82xQjoGna1dvPOY6ltj/pEx59pdgcJfYcynYqMkFIk8bhLJJN9/ndIfX29FTQ== + +"@lerna/has-npm-version@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/has-npm-version/-/has-npm-version-4.0.0.tgz#d3fc3292c545eb28bd493b36e6237cf0279f631c" + integrity sha512-LQ3U6XFH8ZmLCsvsgq1zNDqka0Xzjq5ibVN+igAI5ccRWNaUsE/OcmsyMr50xAtNQMYMzmpw5GVLAivT2/YzCg== + dependencies: + "@lerna/child-process" "4.0.0" + semver "^7.3.4" + +"@lerna/import@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/import/-/import-4.0.0.tgz#bde656c4a451fa87ae41733ff8a8da60547c5465" + integrity sha512-FaIhd+4aiBousKNqC7TX1Uhe97eNKf5/SC7c5WZANVWtC7aBWdmswwDt3usrzCNpj6/Wwr9EtEbYROzxKH8ffg== + dependencies: + "@lerna/child-process" "4.0.0" + "@lerna/command" "4.0.0" + "@lerna/prompt" "4.0.0" + "@lerna/pulse-till-done" "4.0.0" + "@lerna/validation-error" "4.0.0" + dedent "^0.7.0" + fs-extra "^9.1.0" + p-map-series "^2.1.0" + +"@lerna/info@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/info/-/info-4.0.0.tgz#b9fb0e479d60efe1623603958a831a88b1d7f1fc" + integrity sha512-8Uboa12kaCSZEn4XRfPz5KU9XXoexSPS4oeYGj76s2UQb1O1GdnEyfjyNWoUl1KlJ2i/8nxUskpXIftoFYH0/Q== + dependencies: + "@lerna/command" "4.0.0" + "@lerna/output" "4.0.0" + envinfo "^7.7.4" + +"@lerna/init@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/init/-/init-4.0.0.tgz#dadff67e6dfb981e8ccbe0e6a310e837962f6c7a" + integrity sha512-wY6kygop0BCXupzWj5eLvTUqdR7vIAm0OgyV9WHpMYQGfs1V22jhztt8mtjCloD/O0nEe4tJhdG62XU5aYmPNQ== + dependencies: + "@lerna/child-process" "4.0.0" + "@lerna/command" "4.0.0" + fs-extra "^9.1.0" + p-map "^4.0.0" + write-json-file "^4.3.0" + +"@lerna/link@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/link/-/link-4.0.0.tgz#c3a38aabd44279d714e90f2451e31b63f0fb65ba" + integrity sha512-KlvPi7XTAcVOByfaLlOeYOfkkDcd+bejpHMCd1KcArcFTwijOwXOVi24DYomIeHvy6HsX/IUquJ4PPUJIeB4+w== + dependencies: + "@lerna/command" "4.0.0" + "@lerna/package-graph" "4.0.0" + "@lerna/symlink-dependencies" "4.0.0" + p-map "^4.0.0" + slash "^3.0.0" + +"@lerna/list@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/list/-/list-4.0.0.tgz#24b4e6995bd73f81c556793fe502b847efd9d1d7" + integrity sha512-L2B5m3P+U4Bif5PultR4TI+KtW+SArwq1i75QZ78mRYxPc0U/piau1DbLOmwrdqr99wzM49t0Dlvl6twd7GHFg== + dependencies: + "@lerna/command" "4.0.0" + "@lerna/filter-options" "4.0.0" + "@lerna/listable" "4.0.0" + "@lerna/output" "4.0.0" + +"@lerna/listable@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/listable/-/listable-4.0.0.tgz#d00d6cb4809b403f2b0374fc521a78e318b01214" + integrity sha512-/rPOSDKsOHs5/PBLINZOkRIX1joOXUXEtyUs5DHLM8q6/RP668x/1lFhw6Dx7/U+L0+tbkpGtZ1Yt0LewCLgeQ== + dependencies: + "@lerna/query-graph" "4.0.0" + chalk "^4.1.0" + columnify "^1.5.4" + +"@lerna/log-packed@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/log-packed/-/log-packed-4.0.0.tgz#95168fe2e26ac6a71e42f4be857519b77e57a09f" + integrity sha512-+dpCiWbdzgMAtpajLToy9PO713IHoE6GV/aizXycAyA07QlqnkpaBNZ8DW84gHdM1j79TWockGJo9PybVhrrZQ== + dependencies: + byte-size "^7.0.0" + columnify "^1.5.4" + has-unicode "^2.0.1" + npmlog "^4.1.2" + +"@lerna/npm-conf@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/npm-conf/-/npm-conf-4.0.0.tgz#b259fd1e1cee2bf5402b236e770140ff9ade7fd2" + integrity sha512-uS7H02yQNq3oejgjxAxqq/jhwGEE0W0ntr8vM3EfpCW1F/wZruwQw+7bleJQ9vUBjmdXST//tk8mXzr5+JXCfw== + dependencies: + config-chain "^1.1.12" + pify "^5.0.0" + +"@lerna/npm-dist-tag@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/npm-dist-tag/-/npm-dist-tag-4.0.0.tgz#d1e99b4eccd3414142f0548ad331bf2d53f3257a" + integrity sha512-F20sg28FMYTgXqEQihgoqSfwmq+Id3zT23CnOwD+XQMPSy9IzyLf1fFVH319vXIw6NF6Pgs4JZN2Qty6/CQXGw== + dependencies: + "@lerna/otplease" "4.0.0" + npm-package-arg "^8.1.0" + npm-registry-fetch "^9.0.0" + npmlog "^4.1.2" + +"@lerna/npm-install@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/npm-install/-/npm-install-4.0.0.tgz#31180be3ab3b7d1818a1a0c206aec156b7094c78" + integrity sha512-aKNxq2j3bCH3eXl3Fmu4D54s/YLL9WSwV8W7X2O25r98wzrO38AUN6AB9EtmAx+LV/SP15et7Yueg9vSaanRWg== + dependencies: + "@lerna/child-process" "4.0.0" + "@lerna/get-npm-exec-opts" "4.0.0" + fs-extra "^9.1.0" + npm-package-arg "^8.1.0" + npmlog "^4.1.2" + signal-exit "^3.0.3" + write-pkg "^4.0.0" + +"@lerna/npm-publish@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/npm-publish/-/npm-publish-4.0.0.tgz#84eb62e876fe949ae1fd62c60804423dbc2c4472" + integrity sha512-vQb7yAPRo5G5r77DRjHITc9piR9gvEKWrmfCH7wkfBnGWEqu7n8/4bFQ7lhnkujvc8RXOsYpvbMQkNfkYibD/w== + dependencies: + "@lerna/otplease" "4.0.0" + "@lerna/run-lifecycle" "4.0.0" + fs-extra "^9.1.0" + libnpmpublish "^4.0.0" + npm-package-arg "^8.1.0" + npmlog "^4.1.2" + pify "^5.0.0" + read-package-json "^3.0.0" + +"@lerna/npm-run-script@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/npm-run-script/-/npm-run-script-4.0.0.tgz#dfebf4f4601442e7c0b5214f9fb0d96c9350743b" + integrity sha512-Jmyh9/IwXJjOXqKfIgtxi0bxi1pUeKe5bD3S81tkcy+kyng/GNj9WSqD5ZggoNP2NP//s4CLDAtUYLdP7CU9rA== + dependencies: + "@lerna/child-process" "4.0.0" + "@lerna/get-npm-exec-opts" "4.0.0" + npmlog "^4.1.2" + +"@lerna/otplease@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/otplease/-/otplease-4.0.0.tgz#84972eb43448f8a1077435ba1c5e59233b725850" + integrity sha512-Sgzbqdk1GH4psNiT6hk+BhjOfIr/5KhGBk86CEfHNJTk9BK4aZYyJD4lpDbDdMjIV4g03G7pYoqHzH765T4fxw== + dependencies: + "@lerna/prompt" "4.0.0" + +"@lerna/output@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/output/-/output-4.0.0.tgz#b1d72215c0e35483e4f3e9994debc82c621851f2" + integrity sha512-Un1sHtO1AD7buDQrpnaYTi2EG6sLF+KOPEAMxeUYG5qG3khTs2Zgzq5WE3dt2N/bKh7naESt20JjIW6tBELP0w== + dependencies: + npmlog "^4.1.2" + +"@lerna/pack-directory@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/pack-directory/-/pack-directory-4.0.0.tgz#8b617db95d20792f043aaaa13a9ccc0e04cb4c74" + integrity sha512-NJrmZNmBHS+5aM+T8N6FVbaKFScVqKlQFJNY2k7nsJ/uklNKsLLl6VhTQBPwMTbf6Tf7l6bcKzpy7aePuq9UiQ== + dependencies: + "@lerna/get-packed" "4.0.0" + "@lerna/package" "4.0.0" + "@lerna/run-lifecycle" "4.0.0" + npm-packlist "^2.1.4" + npmlog "^4.1.2" + tar "^6.1.0" + temp-write "^4.0.0" + +"@lerna/package-graph@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/package-graph/-/package-graph-4.0.0.tgz#16a00253a8ac810f72041481cb46bcee8d8123dd" + integrity sha512-QED2ZCTkfXMKFoTGoccwUzjHtZMSf3UKX14A4/kYyBms9xfFsesCZ6SLI5YeySEgcul8iuIWfQFZqRw+Qrjraw== + dependencies: + "@lerna/prerelease-id-from-version" "4.0.0" + "@lerna/validation-error" "4.0.0" + npm-package-arg "^8.1.0" + npmlog "^4.1.2" + semver "^7.3.4" + +"@lerna/package@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/package/-/package-4.0.0.tgz#1b4c259c4bcff45c876ee1d591a043aacbc0d6b7" + integrity sha512-l0M/izok6FlyyitxiQKr+gZLVFnvxRQdNhzmQ6nRnN9dvBJWn+IxxpM+cLqGACatTnyo9LDzNTOj2Db3+s0s8Q== + dependencies: + load-json-file "^6.2.0" + npm-package-arg "^8.1.0" + write-pkg "^4.0.0" + +"@lerna/prerelease-id-from-version@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/prerelease-id-from-version/-/prerelease-id-from-version-4.0.0.tgz#c7e0676fcee1950d85630e108eddecdd5b48c916" + integrity sha512-GQqguzETdsYRxOSmdFZ6zDBXDErIETWOqomLERRY54f4p+tk4aJjoVdd9xKwehC9TBfIFvlRbL1V9uQGHh1opg== + dependencies: + semver "^7.3.4" + +"@lerna/profiler@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/profiler/-/profiler-4.0.0.tgz#8a53ab874522eae15d178402bff90a14071908e9" + integrity sha512-/BaEbqnVh1LgW/+qz8wCuI+obzi5/vRE8nlhjPzdEzdmWmZXuCKyWSEzAyHOJWw1ntwMiww5dZHhFQABuoFz9Q== + dependencies: + fs-extra "^9.1.0" + npmlog "^4.1.2" + upath "^2.0.1" + +"@lerna/project@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/project/-/project-4.0.0.tgz#ff84893935833533a74deff30c0e64ddb7f0ba6b" + integrity sha512-o0MlVbDkD5qRPkFKlBZsXZjoNTWPyuL58564nSfZJ6JYNmgAptnWPB2dQlAc7HWRZkmnC2fCkEdoU+jioPavbg== + dependencies: + "@lerna/package" "4.0.0" + "@lerna/validation-error" "4.0.0" + cosmiconfig "^7.0.0" + dedent "^0.7.0" + dot-prop "^6.0.1" + glob-parent "^5.1.1" + globby "^11.0.2" + load-json-file "^6.2.0" + npmlog "^4.1.2" + p-map "^4.0.0" + resolve-from "^5.0.0" + write-json-file "^4.3.0" + +"@lerna/prompt@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/prompt/-/prompt-4.0.0.tgz#5ec69a803f3f0db0ad9f221dad64664d3daca41b" + integrity sha512-4Ig46oCH1TH5M7YyTt53fT6TuaKMgqUUaqdgxvp6HP6jtdak6+amcsqB8YGz2eQnw/sdxunx84DfI9XpoLj4bQ== + dependencies: + inquirer "^7.3.3" + npmlog "^4.1.2" + +"@lerna/publish@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/publish/-/publish-4.0.0.tgz#f67011305adeba120066a3b6d984a5bb5fceef65" + integrity sha512-K8jpqjHrChH22qtkytA5GRKIVFEtqBF6JWj1I8dWZtHs4Jywn8yB1jQ3BAMLhqmDJjWJtRck0KXhQQKzDK2UPg== + dependencies: + "@lerna/check-working-tree" "4.0.0" + "@lerna/child-process" "4.0.0" + "@lerna/collect-updates" "4.0.0" + "@lerna/command" "4.0.0" + "@lerna/describe-ref" "4.0.0" + "@lerna/log-packed" "4.0.0" + "@lerna/npm-conf" "4.0.0" + "@lerna/npm-dist-tag" "4.0.0" + "@lerna/npm-publish" "4.0.0" + "@lerna/otplease" "4.0.0" + "@lerna/output" "4.0.0" + "@lerna/pack-directory" "4.0.0" + "@lerna/prerelease-id-from-version" "4.0.0" + "@lerna/prompt" "4.0.0" + "@lerna/pulse-till-done" "4.0.0" + "@lerna/run-lifecycle" "4.0.0" + "@lerna/run-topologically" "4.0.0" + "@lerna/validation-error" "4.0.0" + "@lerna/version" "4.0.0" + fs-extra "^9.1.0" + libnpmaccess "^4.0.1" + npm-package-arg "^8.1.0" + npm-registry-fetch "^9.0.0" + npmlog "^4.1.2" + p-map "^4.0.0" + p-pipe "^3.1.0" + pacote "^11.2.6" + semver "^7.3.4" + +"@lerna/pulse-till-done@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/pulse-till-done/-/pulse-till-done-4.0.0.tgz#04bace7d483a8205c187b806bcd8be23d7bb80a3" + integrity sha512-Frb4F7QGckaybRhbF7aosLsJ5e9WuH7h0KUkjlzSByVycxY91UZgaEIVjS2oN9wQLrheLMHl6SiFY0/Pvo0Cxg== + dependencies: + npmlog "^4.1.2" + +"@lerna/query-graph@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/query-graph/-/query-graph-4.0.0.tgz#09dd1c819ac5ee3f38db23931143701f8a6eef63" + integrity sha512-YlP6yI3tM4WbBmL9GCmNDoeQyzcyg1e4W96y/PKMZa5GbyUvkS2+Jc2kwPD+5KcXou3wQZxSPzR3Te5OenaDdg== + dependencies: + "@lerna/package-graph" "4.0.0" + +"@lerna/resolve-symlink@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/resolve-symlink/-/resolve-symlink-4.0.0.tgz#6d006628a210c9b821964657a9e20a8c9a115e14" + integrity sha512-RtX8VEUzqT+uLSCohx8zgmjc6zjyRlh6i/helxtZTMmc4+6O4FS9q5LJas2uGO2wKvBlhcD6siibGt7dIC3xZA== + dependencies: + fs-extra "^9.1.0" + npmlog "^4.1.2" + read-cmd-shim "^2.0.0" + +"@lerna/rimraf-dir@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/rimraf-dir/-/rimraf-dir-4.0.0.tgz#2edf3b62d4eb0ef4e44e430f5844667d551ec25a" + integrity sha512-QNH9ABWk9mcMJh2/muD9iYWBk1oQd40y6oH+f3wwmVGKYU5YJD//+zMiBI13jxZRtwBx0vmBZzkBkK1dR11cBg== + dependencies: + "@lerna/child-process" "4.0.0" + npmlog "^4.1.2" + path-exists "^4.0.0" + rimraf "^3.0.2" + +"@lerna/run-lifecycle@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/run-lifecycle/-/run-lifecycle-4.0.0.tgz#e648a46f9210a9bcd7c391df6844498cb5079334" + integrity sha512-IwxxsajjCQQEJAeAaxF8QdEixfI7eLKNm4GHhXHrgBu185JcwScFZrj9Bs+PFKxwb+gNLR4iI5rpUdY8Y0UdGQ== + dependencies: + "@lerna/npm-conf" "4.0.0" + npm-lifecycle "^3.1.5" + npmlog "^4.1.2" + +"@lerna/run-topologically@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/run-topologically/-/run-topologically-4.0.0.tgz#af846eeee1a09b0c2be0d1bfb5ef0f7b04bb1827" + integrity sha512-EVZw9hGwo+5yp+VL94+NXRYisqgAlj0jWKWtAIynDCpghRxCE5GMO3xrQLmQgqkpUl9ZxQFpICgYv5DW4DksQA== + dependencies: + "@lerna/query-graph" "4.0.0" + p-queue "^6.6.2" + +"@lerna/run@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/run/-/run-4.0.0.tgz#4bc7fda055a729487897c23579694f6183c91262" + integrity sha512-9giulCOzlMPzcZS/6Eov6pxE9gNTyaXk0Man+iCIdGJNMrCnW7Dme0Z229WWP/UoxDKg71F2tMsVVGDiRd8fFQ== + dependencies: + "@lerna/command" "4.0.0" + "@lerna/filter-options" "4.0.0" + "@lerna/npm-run-script" "4.0.0" + "@lerna/output" "4.0.0" + "@lerna/profiler" "4.0.0" + "@lerna/run-topologically" "4.0.0" + "@lerna/timer" "4.0.0" + "@lerna/validation-error" "4.0.0" + p-map "^4.0.0" + +"@lerna/symlink-binary@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/symlink-binary/-/symlink-binary-4.0.0.tgz#21009f62d53a425f136cb4c1a32c6b2a0cc02d47" + integrity sha512-zualodWC4q1QQc1pkz969hcFeWXOsVYZC5AWVtAPTDfLl+TwM7eG/O6oP+Rr3fFowspxo6b1TQ6sYfDV6HXNWA== + dependencies: + "@lerna/create-symlink" "4.0.0" + "@lerna/package" "4.0.0" + fs-extra "^9.1.0" + p-map "^4.0.0" + +"@lerna/symlink-dependencies@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/symlink-dependencies/-/symlink-dependencies-4.0.0.tgz#8910eca084ae062642d0490d8972cf2d98e9ebbd" + integrity sha512-BABo0MjeUHNAe2FNGty1eantWp8u83BHSeIMPDxNq0MuW2K3CiQRaeWT3EGPAzXpGt0+hVzBrA6+OT0GPn7Yuw== + dependencies: + "@lerna/create-symlink" "4.0.0" + "@lerna/resolve-symlink" "4.0.0" + "@lerna/symlink-binary" "4.0.0" + fs-extra "^9.1.0" + p-map "^4.0.0" + p-map-series "^2.1.0" + +"@lerna/timer@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/timer/-/timer-4.0.0.tgz#a52e51bfcd39bfd768988049ace7b15c1fd7a6da" + integrity sha512-WFsnlaE7SdOvjuyd05oKt8Leg3ENHICnvX3uYKKdByA+S3g+TCz38JsNs7OUZVt+ba63nC2nbXDlUnuT2Xbsfg== + +"@lerna/validation-error@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/validation-error/-/validation-error-4.0.0.tgz#af9d62fe8304eaa2eb9a6ba1394f9aa807026d35" + integrity sha512-1rBOM5/koiVWlRi3V6dB863E1YzJS8v41UtsHgMr6gB2ncJ2LsQtMKlJpi3voqcgh41H8UsPXR58RrrpPpufyw== + dependencies: + npmlog "^4.1.2" + +"@lerna/version@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/version/-/version-4.0.0.tgz#532659ec6154d8a8789c5ab53878663e244e3228" + integrity sha512-otUgiqs5W9zGWJZSCCMRV/2Zm2A9q9JwSDS7s/tlKq4mWCYriWo7+wsHEA/nPTMDyYyBO5oyZDj+3X50KDUzeA== + dependencies: + "@lerna/check-working-tree" "4.0.0" + "@lerna/child-process" "4.0.0" + "@lerna/collect-updates" "4.0.0" + "@lerna/command" "4.0.0" + "@lerna/conventional-commits" "4.0.0" + "@lerna/github-client" "4.0.0" + "@lerna/gitlab-client" "4.0.0" + "@lerna/output" "4.0.0" + "@lerna/prerelease-id-from-version" "4.0.0" + "@lerna/prompt" "4.0.0" + "@lerna/run-lifecycle" "4.0.0" + "@lerna/run-topologically" "4.0.0" + "@lerna/validation-error" "4.0.0" + chalk "^4.1.0" + dedent "^0.7.0" + load-json-file "^6.2.0" + minimatch "^3.0.4" + npmlog "^4.1.2" + p-map "^4.0.0" + p-pipe "^3.1.0" + p-reduce "^2.1.0" + p-waterfall "^2.1.1" + semver "^7.3.4" + slash "^3.0.0" + temp-write "^4.0.0" + write-json-file "^4.3.0" + +"@lerna/write-log-file@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@lerna/write-log-file/-/write-log-file-4.0.0.tgz#18221a38a6a307d6b0a5844dd592ad53fa27091e" + integrity sha512-XRG5BloiArpXRakcnPHmEHJp+4AtnhRtpDIHSghmXD5EichI1uD73J7FgPp30mm2pDRq3FdqB0NbwSEsJ9xFQg== + dependencies: + npmlog "^4.1.2" + write-file-atomic "^3.0.3" + +"@mattiasbuelens/web-streams-adapter@~0.1.0": + version "0.1.0" + resolved "https://registry.yarnpkg.com/@mattiasbuelens/web-streams-adapter/-/web-streams-adapter-0.1.0.tgz#607b5a25682f4ae2741da7ba6df39302505336b3" + integrity sha512-oV4PyZfwJNtmFWhvlJLqYIX1Nn22ML8FZpS16ZUKv0hg7414xV1fjsGqxQzLT2dyK92TKxsJSwMOd7VNHAtPmA== + +"@mrmlnc/readdir-enhanced@^2.2.1": + version "2.2.1" + resolved "https://registry.yarnpkg.com/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz#524af240d1a360527b730475ecfa1344aa540dde" + integrity sha512-bPHp6Ji8b41szTOcaP63VlnbbO5Ny6dwAATtY6JTjh5N2OLrb5Qk/Th5cRkRQhkWCt+EJsYrNB0MiL+Gpn6e3g== + dependencies: + call-me-maybe "^1.0.1" + glob-to-regexp "^0.3.0" + +"@nodelib/fs.scandir@2.1.5": + version "2.1.5" + resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" + integrity sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g== + dependencies: + "@nodelib/fs.stat" "2.0.5" + run-parallel "^1.1.9" + +"@nodelib/fs.stat@2.0.5", "@nodelib/fs.stat@^2.0.2": + version "2.0.5" + resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz#5bd262af94e9d25bd1e71b05deed44876a222e8b" + integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A== + +"@nodelib/fs.stat@^1.1.2": + version "1.1.3" + resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz#2b5a3ab3f918cca48a8c754c08168e3f03eba61b" + integrity sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw== + +"@nodelib/fs.walk@^1.2.3": + version "1.2.8" + resolved "https://registry.yarnpkg.com/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz#e95737e8bb6746ddedf69c556953494f196fe69a" + integrity sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg== + dependencies: + "@nodelib/fs.scandir" "2.1.5" + fastq "^1.6.0" + +"@npmcli/ci-detect@^1.0.0": + version "1.3.0" + resolved "https://registry.yarnpkg.com/@npmcli/ci-detect/-/ci-detect-1.3.0.tgz#6c1d2c625fb6ef1b9dea85ad0a5afcbef85ef22a" + integrity sha512-oN3y7FAROHhrAt7Rr7PnTSwrHrZVRTS2ZbyxeQwSSYD0ifwM3YNgQqbaRmjcWoPyq77MjchusjJDspbzMmip1Q== + +"@npmcli/git@^2.1.0": + version "2.1.0" + resolved "https://registry.yarnpkg.com/@npmcli/git/-/git-2.1.0.tgz#2fbd77e147530247d37f325930d457b3ebe894f6" + integrity sha512-/hBFX/QG1b+N7PZBFs0bi+evgRZcK9nWBxQKZkGoXUT5hJSwl5c4d7y8/hm+NQZRPhQ67RzFaj5UM9YeyKoryw== + dependencies: + "@npmcli/promise-spawn" "^1.3.2" + lru-cache "^6.0.0" + mkdirp "^1.0.4" + npm-pick-manifest "^6.1.1" + promise-inflight "^1.0.1" + promise-retry "^2.0.1" + semver "^7.3.5" + which "^2.0.2" + +"@npmcli/installed-package-contents@^1.0.6": + version "1.0.7" + resolved "https://registry.yarnpkg.com/@npmcli/installed-package-contents/-/installed-package-contents-1.0.7.tgz#ab7408c6147911b970a8abe261ce512232a3f4fa" + integrity sha512-9rufe0wnJusCQoLpV9ZPKIVP55itrM5BxOXs10DmdbRfgWtHy1LDyskbwRnBghuB0PrF7pNPOqREVtpz4HqzKw== + dependencies: + npm-bundled "^1.1.1" + npm-normalize-package-bin "^1.0.1" + +"@npmcli/move-file@^1.0.1": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@npmcli/move-file/-/move-file-1.1.2.tgz#1a82c3e372f7cae9253eb66d72543d6b8685c674" + integrity sha512-1SUf/Cg2GzGDyaf15aR9St9TWlb+XvbZXWpDx8YKs7MLzMH/BCeopv+y9vzrzgkfykCGuWOlSu3mZhj2+FQcrg== + dependencies: + mkdirp "^1.0.4" + rimraf "^3.0.2" + +"@npmcli/node-gyp@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@npmcli/node-gyp/-/node-gyp-1.0.2.tgz#3cdc1f30e9736dbc417373ed803b42b1a0a29ede" + integrity sha512-yrJUe6reVMpktcvagumoqD9r08fH1iRo01gn1u0zoCApa9lnZGEigVKUd2hzsCId4gdtkZZIVscLhNxMECKgRg== + +"@npmcli/promise-spawn@^1.2.0", "@npmcli/promise-spawn@^1.3.2": + version "1.3.2" + resolved "https://registry.yarnpkg.com/@npmcli/promise-spawn/-/promise-spawn-1.3.2.tgz#42d4e56a8e9274fba180dabc0aea6e38f29274f5" + integrity sha512-QyAGYo/Fbj4MXeGdJcFzZ+FkDkomfRBrPM+9QYJSg+PxgAUL+LU3FneQk37rKR2/zjqkCV1BLHccX98wRXG3Sg== + dependencies: + infer-owner "^1.0.4" + +"@npmcli/run-script@^1.8.2": + version "1.8.5" + resolved "https://registry.yarnpkg.com/@npmcli/run-script/-/run-script-1.8.5.tgz#f250a0c5e1a08a792d775a315d0ff42fc3a51e1d" + integrity sha512-NQspusBCpTjNwNRFMtz2C5MxoxyzlbuJ4YEhxAKrIonTiirKDtatsZictx9RgamQIx6+QuHMNmPl0wQdoESs9A== + dependencies: + "@npmcli/node-gyp" "^1.0.2" + "@npmcli/promise-spawn" "^1.3.2" + infer-owner "^1.0.4" + node-gyp "^7.1.0" + read-package-json-fast "^2.0.1" + +"@octokit/auth-token@^2.4.4": + version "2.4.5" + resolved "https://registry.yarnpkg.com/@octokit/auth-token/-/auth-token-2.4.5.tgz#568ccfb8cb46f36441fac094ce34f7a875b197f3" + integrity sha512-BpGYsPgJt05M7/L/5FoE1PiAbdxXFZkX/3kDYcsvd1v6UhlnE5e96dTDr0ezX/EFwciQxf3cNV0loipsURU+WA== + dependencies: + "@octokit/types" "^6.0.3" + +"@octokit/core@^3.5.0": + version "3.5.1" + resolved "https://registry.yarnpkg.com/@octokit/core/-/core-3.5.1.tgz#8601ceeb1ec0e1b1b8217b960a413ed8e947809b" + integrity sha512-omncwpLVxMP+GLpLPgeGJBF6IWJFjXDS5flY5VbppePYX9XehevbDykRH9PdCdvqt9TS5AOTiDide7h0qrkHjw== + dependencies: + "@octokit/auth-token" "^2.4.4" + "@octokit/graphql" "^4.5.8" + "@octokit/request" "^5.6.0" + "@octokit/request-error" "^2.0.5" + "@octokit/types" "^6.0.3" + before-after-hook "^2.2.0" + universal-user-agent "^6.0.0" + +"@octokit/endpoint@^6.0.1": + version "6.0.12" + resolved "https://registry.yarnpkg.com/@octokit/endpoint/-/endpoint-6.0.12.tgz#3b4d47a4b0e79b1027fb8d75d4221928b2d05658" + integrity sha512-lF3puPwkQWGfkMClXb4k/eUT/nZKQfxinRWJrdZaJO85Dqwo/G0yOC434Jr2ojwafWJMYqFGFa5ms4jJUgujdA== + dependencies: + "@octokit/types" "^6.0.3" + is-plain-object "^5.0.0" + universal-user-agent "^6.0.0" + +"@octokit/graphql@^4.5.8": + version "4.6.4" + resolved "https://registry.yarnpkg.com/@octokit/graphql/-/graphql-4.6.4.tgz#0c3f5bed440822182e972317122acb65d311a5ed" + integrity sha512-SWTdXsVheRmlotWNjKzPOb6Js6tjSqA2a8z9+glDJng0Aqjzti8MEWOtuT8ZSu6wHnci7LZNuarE87+WJBG4vg== + dependencies: + "@octokit/request" "^5.6.0" + "@octokit/types" "^6.0.3" + universal-user-agent "^6.0.0" + +"@octokit/openapi-types@^9.0.0": + version "9.0.0" + resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-9.0.0.tgz#05d33f999326785445c915d25167d68bd5eddb24" + integrity sha512-GSpv5VUFqarOXZl6uWPsDnjChkKCxnaMALmQhzvCWGiMxONQxX7ZwlomCMS+wB1KqxLPCA5n6gYt016oEMkHmQ== + +"@octokit/plugin-enterprise-rest@^6.0.1": + version "6.0.1" + resolved "https://registry.yarnpkg.com/@octokit/plugin-enterprise-rest/-/plugin-enterprise-rest-6.0.1.tgz#e07896739618dab8da7d4077c658003775f95437" + integrity sha512-93uGjlhUD+iNg1iWhUENAtJata6w5nE+V4urXOAlIXdco6xNZtUSfYY8dzp3Udy74aqO/B5UZL80x/YMa5PKRw== + +"@octokit/plugin-paginate-rest@^2.6.2": + version "2.14.0" + resolved "https://registry.yarnpkg.com/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.14.0.tgz#f469cb4a908792fb44679c5973d8bba820c88b0f" + integrity sha512-S2uEu2uHeI7Vf+Lvj8tv3O5/5TCAa8GHS0dUQN7gdM7vKA6ZHAbR6HkAVm5yMb1mbedLEbxOuQ+Fa0SQ7tCDLA== + dependencies: + "@octokit/types" "^6.18.0" + +"@octokit/plugin-request-log@^1.0.2": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@octokit/plugin-request-log/-/plugin-request-log-1.0.4.tgz#5e50ed7083a613816b1e4a28aeec5fb7f1462e85" + integrity sha512-mLUsMkgP7K/cnFEw07kWqXGF5LKrOkD+lhCrKvPHXWDywAwuDUeDwWBpc69XK3pNX0uKiVt8g5z96PJ6z9xCFA== + +"@octokit/plugin-rest-endpoint-methods@5.4.2": + version "5.4.2" + resolved "https://registry.yarnpkg.com/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.4.2.tgz#d090e93ee68ec09985e1ff0a1d2d28581cc883a5" + integrity sha512-imNDDvUMy9YzECcP6zTcKNjwutSwqCYGMZjLPnBHF0kdb3V9URrHWmalD0ZvNEYjwbpm2zw8RPewj3ebCpMBRw== + dependencies: + "@octokit/types" "^6.19.1" + deprecation "^2.3.1" + +"@octokit/request-error@^2.0.5", "@octokit/request-error@^2.1.0": + version "2.1.0" + resolved "https://registry.yarnpkg.com/@octokit/request-error/-/request-error-2.1.0.tgz#9e150357831bfc788d13a4fd4b1913d60c74d677" + integrity sha512-1VIvgXxs9WHSjicsRwq8PlR2LR2x6DwsJAaFgzdi0JfJoGSO8mYI/cHJQ+9FbN21aa+DrgNLnwObmyeSC8Rmpg== + dependencies: + "@octokit/types" "^6.0.3" + deprecation "^2.0.0" + once "^1.4.0" + +"@octokit/request@^5.6.0": + version "5.6.0" + resolved "https://registry.yarnpkg.com/@octokit/request/-/request-5.6.0.tgz#6084861b6e4fa21dc40c8e2a739ec5eff597e672" + integrity sha512-4cPp/N+NqmaGQwbh3vUsYqokQIzt7VjsgTYVXiwpUP2pxd5YiZB2XuTedbb0SPtv9XS7nzAKjAuQxmY8/aZkiA== + dependencies: + "@octokit/endpoint" "^6.0.1" + "@octokit/request-error" "^2.1.0" + "@octokit/types" "^6.16.1" + is-plain-object "^5.0.0" + node-fetch "^2.6.1" + universal-user-agent "^6.0.0" + +"@octokit/rest@^18.1.0": + version "18.6.8" + resolved "https://registry.yarnpkg.com/@octokit/rest/-/rest-18.6.8.tgz#f73ef3b59686df18206183551c2a835d1db1424f" + integrity sha512-n2aT0mJL9N/idCPmnBynCino1qNScfRHvr8OeskQdBNhUYAMc7cxoc8KLlv1DMWxlZUNhed+5kVdu7majVdVag== + dependencies: + "@octokit/core" "^3.5.0" + "@octokit/plugin-paginate-rest" "^2.6.2" + "@octokit/plugin-request-log" "^1.0.2" + "@octokit/plugin-rest-endpoint-methods" "5.4.2" + +"@octokit/types@^6.0.3", "@octokit/types@^6.16.1", "@octokit/types@^6.18.0", "@octokit/types@^6.19.1": + version "6.19.1" + resolved "https://registry.yarnpkg.com/@octokit/types/-/types-6.19.1.tgz#6ea5f759d8d37e892e59c0a65f10892789b84a25" + integrity sha512-hMI2EokQzMG8ABWcnvcrabqQFuFHqUdN0HUOG4DPTaOtnf/jqhzhK1SHOGu5vDlI/x+hWJ60e28VxB7QhOP0CQ== + dependencies: + "@octokit/openapi-types" "^9.0.0" + +"@openpgp/web-stream-tools@0.0.6": + version "0.0.6" + resolved "https://registry.yarnpkg.com/@openpgp/web-stream-tools/-/web-stream-tools-0.0.6.tgz#98ba42f09254149e6a431062f7eab3ebfc804cf7" + integrity sha512-U2Ujy4GUwz315W2QfleOWFnlvXTGz2Fjt4mg/nATedruT3EbIWjWzw4qfbaIvnBHjaVIijltsiESuNxIFRdHkw== + dependencies: + "@mattiasbuelens/web-streams-adapter" "~0.1.0" + web-streams-polyfill "~3.0.3" + +"@sinonjs/commons@^1.7.0": + version "1.8.3" + resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.8.3.tgz#3802ddd21a50a949b6721ddd72da36e67e7f1b2d" + integrity sha512-xkNcLAn/wZaX14RPlwizcKicDk9G3F8m2nU3L7Ukm5zBgTwiT0wsoFAHx9Jq56fJA1z/7uKGtCRu16sOUCLIHQ== + dependencies: + type-detect "4.0.8" + +"@sinonjs/fake-timers@^7.0.2": + version "7.1.2" + resolved "https://registry.yarnpkg.com/@sinonjs/fake-timers/-/fake-timers-7.1.2.tgz#2524eae70c4910edccf99b2f4e6efc5894aff7b5" + integrity sha512-iQADsW4LBMISqZ6Ci1dupJL9pprqwcVFTcOsEmQOEhW+KLCVn/Y4Jrvg2k19fIHCp+iFprriYPTdRcQR8NbUPg== + dependencies: + "@sinonjs/commons" "^1.7.0" + +"@tootallnate/once@1": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" + integrity sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw== + +"@tsconfig/node10@^1.0.7": + version "1.0.8" + resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.8.tgz#c1e4e80d6f964fbecb3359c43bd48b40f7cadad9" + integrity sha512-6XFfSQmMgq0CFLY1MslA/CPUfhIL919M1rMsa5lP2P097N2Wd1sSX0tx1u4olM16fLNhtHZpRhedZJphNJqmZg== + +"@tsconfig/node12@^1.0.7": + version "1.0.9" + resolved "https://registry.yarnpkg.com/@tsconfig/node12/-/node12-1.0.9.tgz#62c1f6dee2ebd9aead80dc3afa56810e58e1a04c" + integrity sha512-/yBMcem+fbvhSREH+s14YJi18sp7J9jpuhYByADT2rypfajMZZN4WQ6zBGgBKp53NKmqI36wFYDb3yaMPurITw== + +"@tsconfig/node14@^1.0.0": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@tsconfig/node14/-/node14-1.0.1.tgz#95f2d167ffb9b8d2068b0b235302fafd4df711f2" + integrity sha512-509r2+yARFfHHE7T6Puu2jjkoycftovhXRqW328PDXTVGKihlb1P8Z9mMZH04ebyajfRY7dedfGynlrFHJUQCg== + +"@tsconfig/node16@^1.0.1": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.2.tgz#423c77877d0569db20e1fc80885ac4118314010e" + integrity sha512-eZxlbI8GZscaGS7kkc/trHTT5xgrjH3/1n2JDwusC9iahPKWMRvRjJSAN5mCXviuTGQ/lHnhvv8Q1YTpnfz9gA== + +"@types/babel__core@^7.0.0", "@types/babel__core@^7.1.14": + version "7.1.15" + resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.15.tgz#2ccfb1ad55a02c83f8e0ad327cbc332f55eb1024" + integrity sha512-bxlMKPDbY8x5h6HBwVzEOk2C8fb6SLfYQ5Jw3uBYuYF1lfWk/kbLd81la82vrIkBb0l+JdmrZaDikPrNxpS/Ew== + dependencies: + "@babel/parser" "^7.1.0" + "@babel/types" "^7.0.0" + "@types/babel__generator" "*" + "@types/babel__template" "*" + "@types/babel__traverse" "*" + +"@types/babel__generator@*": + version "7.6.3" + resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.3.tgz#f456b4b2ce79137f768aa130d2423d2f0ccfaba5" + integrity sha512-/GWCmzJWqV7diQW54smJZzWbSFf4QYtF71WCKhcx6Ru/tFyQIY2eiiITcCAeuPbNSvT9YCGkVMqqvSk2Z0mXiA== + dependencies: + "@babel/types" "^7.0.0" + +"@types/babel__template@*": + version "7.4.1" + resolved "https://registry.yarnpkg.com/@types/babel__template/-/babel__template-7.4.1.tgz#3d1a48fd9d6c0edfd56f2ff578daed48f36c8969" + integrity sha512-azBFKemX6kMg5Io+/rdGT0dkGreboUVR0Cdm3fz9QJWpaQGJRQXl7C+6hOTCZcMll7KFyEQpgbYI2lHdsS4U7g== + dependencies: + "@babel/parser" "^7.1.0" + "@babel/types" "^7.0.0" + +"@types/babel__traverse@*", "@types/babel__traverse@^7.0.4", "@types/babel__traverse@^7.0.6": + version "7.14.2" + resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.14.2.tgz#ffcd470bbb3f8bf30481678fb5502278ca833a43" + integrity sha512-K2waXdXBi2302XUdcHcR1jCeU0LL4TD9HRs/gk0N2Xvrht+G/BfJa4QObBQZfhMdxiCpV3COl5Nfq4uKTeTnJA== + dependencies: + "@babel/types" "^7.3.0" + +"@types/flatbuffers@^1.10.0": + version "1.10.0" + resolved "https://registry.yarnpkg.com/@types/flatbuffers/-/flatbuffers-1.10.0.tgz#aa74e30ffdc86445f2f060e1808fc9d56b5603ba" + integrity sha512-7btbphLrKvo5yl/5CC2OCxUSMx1wV1wvGT1qDXkSt7yi00/YW7E8k6qzXqJHsp+WU0eoG7r6MTQQXI9lIvd0qA== + +"@types/glob@7.1.4", "@types/glob@^7.1.1": + version "7.1.4" + resolved "https://registry.yarnpkg.com/@types/glob/-/glob-7.1.4.tgz#ea59e21d2ee5c517914cb4bc8e4153b99e566672" + integrity sha512-w+LsMxKyYQm347Otw+IfBXOv9UWVjpHpCDdbBMt8Kz/xbvCYNjP+0qPh91Km3iKfSRLBB0P7fAMf0KHrPu+MyA== + dependencies: + "@types/minimatch" "*" + "@types/node" "*" + +"@types/graceful-fs@^4.1.2": + version "4.1.5" + resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.5.tgz#21ffba0d98da4350db64891f92a9e5db3cdb4e15" + integrity sha512-anKkLmZZ+xm4p8JWBf4hElkM4XR+EZeA2M9BAkkTldmcyDY4mbdIJnRghDJH3Ov5ooY7/UAoENtmdMSkaAd7Cw== + dependencies: + "@types/node" "*" + +"@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1": + version "2.0.3" + resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz#4ba8ddb720221f432e443bd5f9117fd22cfd4762" + integrity sha512-sz7iLqvVUg1gIedBOvlkxPlc8/uVzyS5OwGz1cKjXzkl3FpL3al0crU8YGU1WoHkxn0Wxbw5tyi6hvzJKNzFsw== + +"@types/istanbul-lib-report@*": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz#c14c24f18ea8190c118ee7562b7ff99a36552686" + integrity sha512-plGgXAPfVKFoYfa9NpYDAkseG+g6Jr294RqeqcqDixSbU34MZVJRi/P+7Y8GDpzkEwLaGZZOpKIEmeVZNtKsrg== + dependencies: + "@types/istanbul-lib-coverage" "*" + +"@types/istanbul-reports@^3.0.0": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@types/istanbul-reports/-/istanbul-reports-3.0.1.tgz#9153fe98bba2bd565a63add9436d6f0d7f8468ff" + integrity sha512-c3mAZEuK0lvBp8tmuL74XRKn1+y2dcwOUpH7x4WrF6gk1GIgiluDRgMYQtw2OFcBvAJWlt6ASU3tSqxp0Uu0Aw== + dependencies: + "@types/istanbul-lib-report" "*" + +"@types/jest@26.0.24": + version "26.0.24" + resolved "https://registry.yarnpkg.com/@types/jest/-/jest-26.0.24.tgz#943d11976b16739185913a1936e0de0c4a7d595a" + integrity sha512-E/X5Vib8BWqZNRlDxj9vYXhsDwPYbPINqKF9BsnSoon4RQ0D9moEuLD8txgyypFLH7J4+Lho9Nr/c8H0Fi+17w== + dependencies: + jest-diff "^26.0.0" + pretty-format "^26.0.0" + +"@types/json-schema@^7.0.7": + version "7.0.8" + resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.8.tgz#edf1bf1dbf4e04413ca8e5b17b3b7d7d54b59818" + integrity sha512-YSBPTLTVm2e2OoQIDYx8HaeWJ5tTToLH67kXR7zYNGupXMEHa2++G8k+DczX2cFVgalypqtyZIcU19AFcmOpmg== + +"@types/minimatch@*", "@types/minimatch@^3.0.3": + version "3.0.5" + resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40" + integrity sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ== + +"@types/minimist@^1.2.0", "@types/minimist@^1.2.2": + version "1.2.2" + resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.2.tgz#ee771e2ba4b3dc5b372935d549fd9617bf345b8c" + integrity sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ== + +"@types/node@*", "@types/node@^16.4.0": + version "16.4.0" + resolved "https://registry.yarnpkg.com/@types/node/-/node-16.4.0.tgz#2c219eaa3b8d1e4d04f4dd6e40bc68c7467d5272" + integrity sha512-HrJuE7Mlqcjj+00JqMWpZ3tY8w7EUd+S0U3L1+PQSWiXZbOgyQDvi+ogoUxaHApPJq5diKxYBQwA3iIlNcPqOg== + +"@types/node@^13.7.4": + version "13.13.52" + resolved "https://registry.yarnpkg.com/@types/node/-/node-13.13.52.tgz#03c13be70b9031baaed79481c0c0cfb0045e53f7" + integrity sha512-s3nugnZumCC//n4moGGe6tkNMyYEdaDBitVjwPxXmR5lnMG5dHePinH2EdxkG3Rh1ghFHHixAG4NJhpJW1rthQ== + +"@types/normalize-package-data@^2.4.0": + version "2.4.1" + resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301" + integrity sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw== + +"@types/parse-json@^4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" + integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== + +"@types/prettier@^2.1.5": + version "2.3.2" + resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.3.2.tgz#fc8c2825e4ed2142473b4a81064e6e081463d1b3" + integrity sha512-eI5Yrz3Qv4KPUa/nSIAi0h+qX0XyewOliug5F2QAtuRg6Kjg6jfmxe1GIwoIRhZspD1A0RP8ANrPwvEXXtRFog== + +"@types/randomatic@3.1.2": + version "3.1.2" + resolved "https://registry.yarnpkg.com/@types/randomatic/-/randomatic-3.1.2.tgz#3485f0e113bf47fe25fee62fc20ca27713642975" + integrity sha512-lLsR0U1lUTjJ8vy1r3VGWlgprGtB/QPVwxs+QVSe28b0MS/7sR5tUfvhDd9XLV/AWc50OmDADAhzdqujavdykg== + +"@types/stack-utils@^2.0.0": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.1.tgz#20f18294f797f2209b5f65c8e3b5c8e8261d127c" + integrity sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw== + +"@types/yargs-parser@*": + version "20.2.1" + resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-20.2.1.tgz#3b9ce2489919d9e4fea439b76916abc34b2df129" + integrity sha512-7tFImggNeNBVMsn0vLrpn1H1uPrUBdnARPTpZoitY37ZrdJREzf7I16tMrlK3hen349gr1NYh8CmZQa7CTG6Aw== + +"@types/yargs@^15.0.0": + version "15.0.14" + resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-15.0.14.tgz#26d821ddb89e70492160b66d10a0eb6df8f6fb06" + integrity sha512-yEJzHoxf6SyQGhBhIYGXQDSCkJjB6HohDShto7m8vaKg9Yp0Yn8+71J9eakh2bnPg6BfsH9PRMhiRTZnd4eXGQ== + dependencies: + "@types/yargs-parser" "*" + +"@types/yargs@^16.0.0": + version "16.0.4" + resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-16.0.4.tgz#26aad98dd2c2a38e421086ea9ad42b9e51642977" + integrity sha512-T8Yc9wt/5LbJyCaLiHPReJa0kApcIgJ7Bn735GjItUfh08Z1pJvu8QZqb9s+mMvKV6WUQRV7K2R46YbjMXTTJw== + dependencies: + "@types/yargs-parser" "*" + +"@typescript-eslint/eslint-plugin@4.28.4": + version "4.28.4" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.28.4.tgz#e73c8cabbf3f08dee0e1bda65ed4e622ae8f8921" + integrity sha512-s1oY4RmYDlWMlcV0kKPBaADn46JirZzvvH7c2CtAqxCY96S538JRBAzt83RrfkDheV/+G/vWNK0zek+8TB3Gmw== + dependencies: + "@typescript-eslint/experimental-utils" "4.28.4" + "@typescript-eslint/scope-manager" "4.28.4" + debug "^4.3.1" + functional-red-black-tree "^1.0.1" + regexpp "^3.1.0" + semver "^7.3.5" + tsutils "^3.21.0" + +"@typescript-eslint/experimental-utils@4.28.4", "@typescript-eslint/experimental-utils@^4.0.1": + version "4.28.4" + resolved "https://registry.yarnpkg.com/@typescript-eslint/experimental-utils/-/experimental-utils-4.28.4.tgz#9c70c35ebed087a5c70fb0ecd90979547b7fec96" + integrity sha512-OglKWOQRWTCoqMSy6pm/kpinEIgdcXYceIcH3EKWUl4S8xhFtN34GQRaAvTIZB9DD94rW7d/U7tUg3SYeDFNHA== + dependencies: + "@types/json-schema" "^7.0.7" + "@typescript-eslint/scope-manager" "4.28.4" + "@typescript-eslint/types" "4.28.4" + "@typescript-eslint/typescript-estree" "4.28.4" + eslint-scope "^5.1.1" + eslint-utils "^3.0.0" + +"@typescript-eslint/parser@4.28.4": + version "4.28.4" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-4.28.4.tgz#bc462dc2779afeefdcf49082516afdc3e7b96fab" + integrity sha512-4i0jq3C6n+og7/uCHiE6q5ssw87zVdpUj1k6VlVYMonE3ILdFApEzTWgppSRG4kVNB/5jxnH+gTeKLMNfUelQA== + dependencies: + "@typescript-eslint/scope-manager" "4.28.4" + "@typescript-eslint/types" "4.28.4" + "@typescript-eslint/typescript-estree" "4.28.4" + debug "^4.3.1" + +"@typescript-eslint/scope-manager@4.28.4": + version "4.28.4" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-4.28.4.tgz#bdbce9b6a644e34f767bd68bc17bb14353b9fe7f" + integrity sha512-ZJBNs4usViOmlyFMt9X9l+X0WAFcDH7EdSArGqpldXu7aeZxDAuAzHiMAeI+JpSefY2INHrXeqnha39FVqXb8w== + dependencies: + "@typescript-eslint/types" "4.28.4" + "@typescript-eslint/visitor-keys" "4.28.4" + +"@typescript-eslint/types@4.28.4": + version "4.28.4" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-4.28.4.tgz#41acbd79b5816b7c0dd7530a43d97d020d3aeb42" + integrity sha512-3eap4QWxGqkYuEmVebUGULMskR6Cuoc/Wii0oSOddleP4EGx1tjLnZQ0ZP33YRoMDCs5O3j56RBV4g14T4jvww== + +"@typescript-eslint/typescript-estree@4.28.4": + version "4.28.4" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-4.28.4.tgz#252e6863278dc0727244be9e371eb35241c46d00" + integrity sha512-z7d8HK8XvCRyN2SNp+OXC2iZaF+O2BTquGhEYLKLx5k6p0r05ureUtgEfo5f6anLkhCxdHtCf6rPM1p4efHYDQ== + dependencies: + "@typescript-eslint/types" "4.28.4" + "@typescript-eslint/visitor-keys" "4.28.4" + debug "^4.3.1" + globby "^11.0.3" + is-glob "^4.0.1" + semver "^7.3.5" + tsutils "^3.21.0" + +"@typescript-eslint/visitor-keys@4.28.4": + version "4.28.4" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-4.28.4.tgz#92dacfefccd6751cbb0a964f06683bfd72d0c4d3" + integrity sha512-NIAXAdbz1XdOuzqkJHjNKXKj8QQ4cv5cxR/g0uQhCYf/6//XrmfpaYsM7PnBcNbfvTDLUkqQ5TPNm1sozDdTWg== + dependencies: + "@typescript-eslint/types" "4.28.4" + eslint-visitor-keys "^2.0.0" + +JSONStream@^1.0.4: + version "1.3.5" + resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.5.tgz#3208c1f08d3a4d99261ab64f92302bc15e111ca0" + integrity sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ== + dependencies: + jsonparse "^1.2.0" + through ">=2.2.7 <3" + +abab@^2.0.3, abab@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.5.tgz#c0b678fb32d60fc1219c784d6a826fe385aeb79a" + integrity sha512-9IK9EadsbHo6jLWIpxpR6pL0sazTXV6+SQv25ZB+F7Bj9mJNaOc4nCRabwd5M/JwmUa8idz6Eci6eKfJryPs6Q== + +abbrev@1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" + integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== + +acorn-globals@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/acorn-globals/-/acorn-globals-6.0.0.tgz#46cdd39f0f8ff08a876619b55f5ac8a6dc770b45" + integrity sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg== + dependencies: + acorn "^7.1.1" + acorn-walk "^7.1.1" + +acorn-jsx@^5.3.1: + version "5.3.2" + resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" + integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== + +acorn-walk@^7.1.1: + version "7.2.0" + resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc" + integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA== + +acorn@^6.4.1: + version "6.4.2" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-6.4.2.tgz#35866fd710528e92de10cf06016498e47e39e1e6" + integrity sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ== + +acorn@^7.1.1, acorn@^7.4.0: + version "7.4.1" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" + integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== + +acorn@^8.2.4: + version "8.4.1" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.4.1.tgz#56c36251fc7cabc7096adc18f05afe814321a28c" + integrity sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA== + +add-stream@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/add-stream/-/add-stream-1.0.0.tgz#6a7990437ca736d5e1288db92bd3266d5f5cb2aa" + integrity sha1-anmQQ3ynNtXhKI25K9MmbV9csqo= + +agent-base@6, agent-base@^6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" + integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== + dependencies: + debug "4" + +agentkeepalive@^4.1.3: + version "4.1.4" + resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.1.4.tgz#d928028a4862cb11718e55227872e842a44c945b" + integrity sha512-+V/rGa3EuU74H6wR04plBb7Ks10FbtUQgRj/FQOG7uUIEuaINI+AiqJR1k6t3SVNs7o7ZjIdus6706qqzVq8jQ== + dependencies: + debug "^4.1.0" + depd "^1.1.2" + humanize-ms "^1.2.1" + +aggregate-error@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-3.1.0.tgz#92670ff50f5359bdb7a3e0d40d0ec30c5737687a" + integrity sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA== + dependencies: + clean-stack "^2.0.0" + indent-string "^4.0.0" + +ajv@^6.10.0, ajv@^6.12.3, ajv@^6.12.4: + version "6.12.6" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" + integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== + dependencies: + fast-deep-equal "^3.1.1" + fast-json-stable-stringify "^2.0.0" + json-schema-traverse "^0.4.1" + uri-js "^4.2.2" + +ajv@^8.0.1: + version "8.6.2" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.6.2.tgz#2fb45e0e5fcbc0813326c1c3da535d1881bb0571" + integrity sha512-9807RlWAgT564wT+DjeyU5OFMPjmzxVobvDFmNAhY+5zD6A2ly3jDp6sgnfyDtlIQ+7H97oc/DGCzzfu9rjw9w== + dependencies: + fast-deep-equal "^3.1.1" + json-schema-traverse "^1.0.0" + require-from-string "^2.0.2" + uri-js "^4.2.2" + +ansi-colors@^1.0.1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-1.1.0.tgz#6374b4dd5d4718ff3ce27a671a3b1cad077132a9" + integrity sha512-SFKX67auSNoVR38N3L+nvsPjOE0bybKTYbkf5tRvushrAPQ9V75huw0ZxBkKVeRU9kqH3d6HA4xTckbwZ4ixmA== + dependencies: + ansi-wrap "^0.1.0" + +ansi-colors@^3.0.5: + version "3.2.4" + resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-3.2.4.tgz#e3a3da4bfbae6c86a9c285625de124a234026fbf" + integrity sha512-hHUXGagefjN2iRrID63xckIvotOXOojhQKWIPUZ4mNUZ9nLZW+7FMNoE1lOkEhNWYsx/7ysGIuJYCiMAA9FnrA== + +ansi-colors@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-4.1.1.tgz#cbb9ae256bf750af1eab344f229aa27fe94ba348" + integrity sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA== + +ansi-escapes@^4.2.1, ansi-escapes@^4.3.0: + version "4.3.2" + resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-4.3.2.tgz#6b2291d1db7d98b6521d5f1efa42d0f3a9feb65e" + integrity sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ== + dependencies: + type-fest "^0.21.3" + +ansi-gray@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/ansi-gray/-/ansi-gray-0.1.1.tgz#2962cf54ec9792c48510a3deb524436861ef7251" + integrity sha1-KWLPVOyXksSFEKPetSRDaGHvclE= + dependencies: + ansi-wrap "0.1.0" + +ansi-regex@^2.0.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" + integrity sha1-w7M6te42DYbg5ijwRorn7yfWVN8= + +ansi-regex@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.0.tgz#ed0317c322064f79466c02966bddb605ab37d998" + integrity sha1-7QMXwyIGT3lGbAKWa922Bas32Zg= + +ansi-regex@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" + integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== + +ansi-styles@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" + integrity sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA== + dependencies: + color-convert "^1.9.0" + +ansi-styles@^4.0.0, ansi-styles@^4.1.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" + integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== + dependencies: + color-convert "^2.0.1" + +ansi-styles@^5.0.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-5.2.0.tgz#07449690ad45777d1924ac2abb2fc8895dba836b" + integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA== + +ansi-wrap@0.1.0, ansi-wrap@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/ansi-wrap/-/ansi-wrap-0.1.0.tgz#a82250ddb0015e9a27ca82e82ea603bbfa45efaf" + integrity sha1-qCJQ3bABXponyoLoLqYDu/pF768= + +anymatch@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-2.0.0.tgz#bcb24b4f37934d9aa7ac17b4adaf89e7c76ef2eb" + integrity sha512-5teOsQWABXHHBFP9y3skS5P3d/WfWXpv3FUpy+LorMrNYaT9pI4oLMQX7jzQ2KklNpGpWHzdCXTDT2Y3XGlZBw== + dependencies: + micromatch "^3.1.4" + normalize-path "^2.1.1" + +anymatch@^3.0.3: + version "3.1.2" + resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.2.tgz#c0557c096af32f106198f4f4e2a383537e378716" + integrity sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg== + dependencies: + normalize-path "^3.0.0" + picomatch "^2.0.4" + +append-buffer@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/append-buffer/-/append-buffer-1.0.2.tgz#d8220cf466081525efea50614f3de6514dfa58f1" + integrity sha1-2CIM9GYIFSXv6lBhTz3mUU36WPE= + dependencies: + buffer-equal "^1.0.0" + +aproba@^1.0.3: + version "1.2.0" + resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" + integrity sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw== + +aproba@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/aproba/-/aproba-2.0.0.tgz#52520b8ae5b569215b354efc0caa3fe1e45a8adc" + integrity sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ== + +archy@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/archy/-/archy-1.0.0.tgz#f9c8c13757cc1dd7bc379ac77b2c62a5c2868c40" + integrity sha1-+cjBN1fMHde8N5rHeyxipcKGjEA= + +are-we-there-yet@~1.1.2: + version "1.1.5" + resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz#4b35c2944f062a8bfcda66410760350fe9ddfc21" + integrity sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w== + dependencies: + delegates "^1.0.0" + readable-stream "^2.0.6" + +arg@^4.1.0: + version "4.1.3" + resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089" + integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA== + +argparse@^1.0.7: + version "1.0.10" + resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" + integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== + dependencies: + sprintf-js "~1.0.2" + +arr-diff@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/arr-diff/-/arr-diff-4.0.0.tgz#d6461074febfec71e7e15235761a329a5dc7c520" + integrity sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA= + +arr-filter@^1.1.1: + version "1.1.2" + resolved "https://registry.yarnpkg.com/arr-filter/-/arr-filter-1.1.2.tgz#43fdddd091e8ef11aa4c45d9cdc18e2dff1711ee" + integrity sha1-Q/3d0JHo7xGqTEXZzcGOLf8XEe4= + dependencies: + make-iterator "^1.0.0" + +arr-flatten@^1.0.1, arr-flatten@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/arr-flatten/-/arr-flatten-1.1.0.tgz#36048bbff4e7b47e136644316c99669ea5ae91f1" + integrity sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg== + +arr-map@^2.0.0, arr-map@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/arr-map/-/arr-map-2.0.2.tgz#3a77345ffc1cf35e2a91825601f9e58f2e24cac4" + integrity sha1-Onc0X/wc814qkYJWAfnljy4kysQ= + dependencies: + make-iterator "^1.0.0" + +arr-union@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/arr-union/-/arr-union-3.1.0.tgz#e39b09aea9def866a8f206e288af63919bae39c4" + integrity sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ= + +array-back@^3.0.1, array-back@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/array-back/-/array-back-3.1.0.tgz#b8859d7a508871c9a7b2cf42f99428f65e96bfb0" + integrity sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q== + +array-back@^4.0.1: + version "4.0.2" + resolved "https://registry.yarnpkg.com/array-back/-/array-back-4.0.2.tgz#8004e999a6274586beeb27342168652fdb89fa1e" + integrity sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg== + +array-differ@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/array-differ/-/array-differ-3.0.0.tgz#3cbb3d0f316810eafcc47624734237d6aee4ae6b" + integrity sha512-THtfYS6KtME/yIAhKjZ2ul7XI96lQGHRputJQHO80LAWQnuGP4iCIN8vdMRboGbIEYBwU33q8Tch1os2+X0kMg== + +array-each@^1.0.0, array-each@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/array-each/-/array-each-1.0.1.tgz#a794af0c05ab1752846ee753a1f211a05ba0c44f" + integrity sha1-p5SvDAWrF1KEbudTofIRoFugxE8= + +array-ify@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/array-ify/-/array-ify-1.0.0.tgz#9e528762b4a9066ad163a6962a364418e9626ece" + integrity sha1-nlKHYrSpBmrRY6aWKjZEGOlibs4= + +array-initial@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/array-initial/-/array-initial-1.1.0.tgz#2fa74b26739371c3947bd7a7adc73be334b3d795" + integrity sha1-L6dLJnOTccOUe9enrcc74zSz15U= + dependencies: + array-slice "^1.0.0" + is-number "^4.0.0" + +array-last@^1.1.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/array-last/-/array-last-1.3.0.tgz#7aa77073fec565ddab2493f5f88185f404a9d336" + integrity sha512-eOCut5rXlI6aCOS7Z7kCplKRKyiFQ6dHFBem4PwlwKeNFk2/XxTrhRh5T9PyaEWGy/NHTZWbY+nsZlNFJu9rYg== + dependencies: + is-number "^4.0.0" + +array-slice@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/array-slice/-/array-slice-1.1.0.tgz#e368ea15f89bc7069f7ffb89aec3a6c7d4ac22d4" + integrity sha512-B1qMD3RBP7O8o0H2KbrXDyB0IccejMF15+87Lvlor12ONPRHP6gTjXMNkt/d3ZuOGbAe66hFmaCfECI24Ufp6w== + +array-sort@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/array-sort/-/array-sort-1.0.0.tgz#e4c05356453f56f53512a7d1d6123f2c54c0a88a" + integrity sha512-ihLeJkonmdiAsD7vpgN3CRcx2J2S0TiYW+IS/5zHBI7mKUq3ySvBdzzBfD236ubDBQFiiyG3SWCPc+msQ9KoYg== + dependencies: + default-compare "^1.0.0" + get-value "^2.0.6" + kind-of "^5.0.2" + +array-union@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/array-union/-/array-union-1.0.2.tgz#9a34410e4f4e3da23dea375be5be70f24778ec39" + integrity sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk= + dependencies: + array-uniq "^1.0.1" + +array-union@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" + integrity sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw== + +array-uniq@^1.0.1: + version "1.0.3" + resolved "https://registry.yarnpkg.com/array-uniq/-/array-uniq-1.0.3.tgz#af6ac877a25cc7f74e058894753858dfdb24fdb6" + integrity sha1-r2rId6Jcx/dOBYiUdThY39sk/bY= + +array-unique@^0.3.2: + version "0.3.2" + resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428" + integrity sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg= + +arrify@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d" + integrity sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0= + +arrify@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" + integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== + +asap@^2.0.0, asap@~2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" + integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY= + +asn1@~0.2.3: + version "0.2.4" + resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136" + integrity sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg== + dependencies: + safer-buffer "~2.1.0" + +assert-plus@1.0.0, assert-plus@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" + integrity sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU= + +assign-symbols@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/assign-symbols/-/assign-symbols-1.0.0.tgz#59667f41fadd4f20ccbc2bb96b8d4f7f78ec0367" + integrity sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c= + +astral-regex@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/astral-regex/-/astral-regex-2.0.0.tgz#483143c567aeed4785759c0865786dc77d7d2e31" + integrity sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ== + +async-done@1.3.2, async-done@^1.2.0, async-done@^1.2.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/async-done/-/async-done-1.3.2.tgz#5e15aa729962a4b07414f528a88cdf18e0b290a2" + integrity sha512-uYkTP8dw2og1tu1nmza1n1CMW0qb8gWWlwqMmLb7MhBVs4BXrFziT6HXUd+/RlRA/i4H9AkofYloUbs1fwMqlw== + dependencies: + end-of-stream "^1.1.0" + once "^1.3.2" + process-nextick-args "^2.0.0" + stream-exhaust "^1.0.1" + +async-each@^1.0.1: + version "1.0.3" + resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.3.tgz#b727dbf87d7651602f06f4d4ac387f47d91b0cbf" + integrity sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ== + +async-settle@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/async-settle/-/async-settle-1.0.0.tgz#1d0a914bb02575bec8a8f3a74e5080f72b2c0c6b" + integrity sha1-HQqRS7Aldb7IqPOnTlCA9yssDGs= + dependencies: + async-done "^1.2.2" + +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= + +at-least-node@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/at-least-node/-/at-least-node-1.0.0.tgz#602cd4b46e844ad4effc92a8011a3c46e0238dc2" + integrity sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg== + +atob@^2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9" + integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg== + +aws-sign2@~0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" + integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg= + +aws4@^1.8.0: + version "1.11.0" + resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" + integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== + +babel-jest@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-27.0.6.tgz#e99c6e0577da2655118e3608b68761a5a69bd0d8" + integrity sha512-iTJyYLNc4wRofASmofpOc5NK9QunwMk+TLFgGXsTFS8uEqmd8wdI7sga0FPe2oVH3b5Agt/EAK1QjPEuKL8VfA== + dependencies: + "@jest/transform" "^27.0.6" + "@jest/types" "^27.0.6" + "@types/babel__core" "^7.1.14" + babel-plugin-istanbul "^6.0.0" + babel-preset-jest "^27.0.6" + chalk "^4.0.0" + graceful-fs "^4.2.4" + slash "^3.0.0" + +babel-plugin-istanbul@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/babel-plugin-istanbul/-/babel-plugin-istanbul-6.0.0.tgz#e159ccdc9af95e0b570c75b4573b7c34d671d765" + integrity sha512-AF55rZXpe7trmEylbaE1Gv54wn6rwU03aptvRoVIGP8YykoSxqdVLV1TfwflBCE/QtHmqtP8SWlTENqbK8GCSQ== + dependencies: + "@babel/helper-plugin-utils" "^7.0.0" + "@istanbuljs/load-nyc-config" "^1.0.0" + "@istanbuljs/schema" "^0.1.2" + istanbul-lib-instrument "^4.0.0" + test-exclude "^6.0.0" + +babel-plugin-jest-hoist@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-27.0.6.tgz#f7c6b3d764af21cb4a2a1ab6870117dbde15b456" + integrity sha512-CewFeM9Vv2gM7Yr9n5eyyLVPRSiBnk6lKZRjgwYnGKSl9M14TMn2vkN02wTF04OGuSDLEzlWiMzvjXuW9mB6Gw== + dependencies: + "@babel/template" "^7.3.3" + "@babel/types" "^7.3.3" + "@types/babel__core" "^7.0.0" + "@types/babel__traverse" "^7.0.6" + +babel-preset-current-node-syntax@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.0.1.tgz#b4399239b89b2a011f9ddbe3e4f401fc40cff73b" + integrity sha512-M7LQ0bxarkxQoN+vz5aJPsLBn77n8QgTFmo8WK0/44auK2xlCXrYcUxHFxgU7qW5Yzw/CjmLRK2uJzaCd7LvqQ== + dependencies: + "@babel/plugin-syntax-async-generators" "^7.8.4" + "@babel/plugin-syntax-bigint" "^7.8.3" + "@babel/plugin-syntax-class-properties" "^7.8.3" + "@babel/plugin-syntax-import-meta" "^7.8.3" + "@babel/plugin-syntax-json-strings" "^7.8.3" + "@babel/plugin-syntax-logical-assignment-operators" "^7.8.3" + "@babel/plugin-syntax-nullish-coalescing-operator" "^7.8.3" + "@babel/plugin-syntax-numeric-separator" "^7.8.3" + "@babel/plugin-syntax-object-rest-spread" "^7.8.3" + "@babel/plugin-syntax-optional-catch-binding" "^7.8.3" + "@babel/plugin-syntax-optional-chaining" "^7.8.3" + "@babel/plugin-syntax-top-level-await" "^7.8.3" + +babel-preset-jest@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/babel-preset-jest/-/babel-preset-jest-27.0.6.tgz#909ef08e9f24a4679768be2f60a3df0856843f9d" + integrity sha512-WObA0/Biw2LrVVwZkF/2GqbOdzhKD6Fkdwhoy9ASIrOWr/zodcSpQh72JOkEn6NWyjmnPDjNSqaGN4KnpKzhXw== + dependencies: + babel-plugin-jest-hoist "^27.0.6" + babel-preset-current-node-syntax "^1.0.0" + +bach@^1.0.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/bach/-/bach-1.2.0.tgz#4b3ce96bf27134f79a1b414a51c14e34c3bd9880" + integrity sha1-Szzpa/JxNPeaG0FKUcFONMO9mIA= + dependencies: + arr-filter "^1.1.1" + arr-flatten "^1.0.1" + arr-map "^2.0.0" + array-each "^1.0.0" + array-initial "^1.0.0" + array-last "^1.1.1" + async-done "^1.2.2" + async-settle "^1.0.0" + now-and-later "^2.0.0" + +balanced-match@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" + integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== + +base@^0.11.1: + version "0.11.2" + resolved "https://registry.yarnpkg.com/base/-/base-0.11.2.tgz#7bde5ced145b6d551a90db87f83c558b4eb48a8f" + integrity sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg== + dependencies: + cache-base "^1.0.1" + class-utils "^0.3.5" + component-emitter "^1.2.1" + define-property "^1.0.0" + isobject "^3.0.1" + mixin-deep "^1.2.0" + pascalcase "^0.1.1" + +bcrypt-pbkdf@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" + integrity sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4= + dependencies: + tweetnacl "^0.14.3" + +before-after-hook@^2.2.0: + version "2.2.2" + resolved "https://registry.yarnpkg.com/before-after-hook/-/before-after-hook-2.2.2.tgz#a6e8ca41028d90ee2c24222f201c90956091613e" + integrity sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ== + +benchmark@^2.1.4: + version "2.1.4" + resolved "https://registry.yarnpkg.com/benchmark/-/benchmark-2.1.4.tgz#09f3de31c916425d498cc2ee565a0ebf3c2a5629" + integrity sha1-CfPeMckWQl1JjMLuVloOvzwqVik= + dependencies: + lodash "^4.17.4" + platform "^1.3.3" + +benny@3.6.15: + version "3.6.15" + resolved "https://registry.yarnpkg.com/benny/-/benny-3.6.15.tgz#930826819b89546b274febe803da2d248a676caa" + integrity sha512-kq6XVGGYVou3Y8KNPs3SEF881vi5fJ8sIf9w69D2rreiNfRicWVWK6u6/mObMw6BiexoHHumtipn5gcu0Tngng== + dependencies: + "@arrows/composition" "^1.0.0" + "@arrows/dispatch" "^1.0.2" + "@arrows/multimethod" "^1.1.6" + benchmark "^2.1.4" + fs-extra "^9.0.1" + json2csv "^5.0.4" + kleur "^4.1.3" + log-update "^4.0.0" + prettier "^2.1.2" + stats-median "^1.0.1" + +binary-extensions@^1.0.0: + version "1.13.1" + resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.13.1.tgz#598afe54755b2868a5330d2aff9d4ebb53209b65" + integrity sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw== + +bindings@^1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" + integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ== + dependencies: + file-uri-to-path "1.0.0" + +brace-expansion@^1.1.7: + version "1.1.11" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" + integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== + dependencies: + balanced-match "^1.0.0" + concat-map "0.0.1" + +braces@^2.3.1, braces@^2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/braces/-/braces-2.3.2.tgz#5979fd3f14cd531565e5fa2df1abfff1dfaee729" + integrity sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w== + dependencies: + arr-flatten "^1.1.0" + array-unique "^0.3.2" + extend-shallow "^2.0.1" + fill-range "^4.0.0" + isobject "^3.0.1" + repeat-element "^1.1.2" + snapdragon "^0.8.1" + snapdragon-node "^2.0.1" + split-string "^3.0.2" + to-regex "^3.0.1" + +braces@^3.0.1: + version "3.0.2" + resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107" + integrity sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A== + dependencies: + fill-range "^7.0.1" + +browser-process-hrtime@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz#3c9b4b7d782c8121e56f10106d84c0d0ffc94626" + integrity sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow== + +browserslist@^4.16.6: + version "4.16.6" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.16.6.tgz#d7901277a5a88e554ed305b183ec9b0c08f66fa2" + integrity sha512-Wspk/PqO+4W9qp5iUTJsa1B/QrYn1keNCcEP5OvP7WBwT4KaDly0uONYmC6Xa3Z5IqnUgS0KcgLYu1l74x0ZXQ== + dependencies: + caniuse-lite "^1.0.30001219" + colorette "^1.2.2" + electron-to-chromium "^1.3.723" + escalade "^3.1.1" + node-releases "^1.1.71" + +bs-logger@0.x: + version "0.2.6" + resolved "https://registry.yarnpkg.com/bs-logger/-/bs-logger-0.2.6.tgz#eb7d365307a72cf974cc6cda76b68354ad336bd8" + integrity sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog== + dependencies: + fast-json-stable-stringify "2.x" + +bser@2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/bser/-/bser-2.1.1.tgz#e6787da20ece9d07998533cfd9de6f5c38f4bc05" + integrity sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ== + dependencies: + node-int64 "^0.4.0" + +buffer-equal@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/buffer-equal/-/buffer-equal-1.0.0.tgz#59616b498304d556abd466966b22eeda3eca5fbe" + integrity sha1-WWFrSYME1Var1GaWayLu2j7KX74= + +buffer-from@1.x, buffer-from@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" + integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== + +builtins@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/builtins/-/builtins-1.0.3.tgz#cb94faeb61c8696451db36534e1422f94f0aee88" + integrity sha1-y5T662HIaWRR2zZTThQi+U8K7og= + +byline@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/byline/-/byline-5.0.0.tgz#741c5216468eadc457b03410118ad77de8c1ddb1" + integrity sha1-dBxSFkaOrcRXsDQQEYrXfejB3bE= + +byte-size@^7.0.0: + version "7.0.1" + resolved "https://registry.yarnpkg.com/byte-size/-/byte-size-7.0.1.tgz#b1daf3386de7ab9d706b941a748dbfc71130dee3" + integrity sha512-crQdqyCwhokxwV1UyDzLZanhkugAgft7vt0qbbdt60C6Zf3CAiGmtUCylbtYwrU6loOUw3euGrNtW1J651ot1A== + +cacache@^15.0.5, cacache@^15.2.0: + version "15.2.0" + resolved "https://registry.yarnpkg.com/cacache/-/cacache-15.2.0.tgz#73af75f77c58e72d8c630a7a2858cb18ef523389" + integrity sha512-uKoJSHmnrqXgthDFx/IU6ED/5xd+NNGe+Bb+kLZy7Ku4P+BaiWEUflAKPZ7eAzsYGcsAGASJZsybXp+quEcHTw== + dependencies: + "@npmcli/move-file" "^1.0.1" + chownr "^2.0.0" + fs-minipass "^2.0.0" + glob "^7.1.4" + infer-owner "^1.0.4" + lru-cache "^6.0.0" + minipass "^3.1.1" + minipass-collect "^1.0.2" + minipass-flush "^1.0.5" + minipass-pipeline "^1.2.2" + mkdirp "^1.0.3" + p-map "^4.0.0" + promise-inflight "^1.0.1" + rimraf "^3.0.2" + ssri "^8.0.1" + tar "^6.0.2" + unique-filename "^1.1.1" + +cache-base@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/cache-base/-/cache-base-1.0.1.tgz#0a7f46416831c8b662ee36fe4e7c59d76f666ab2" + integrity sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ== + dependencies: + collection-visit "^1.0.0" + component-emitter "^1.2.1" + get-value "^2.0.6" + has-value "^1.0.0" + isobject "^3.0.1" + set-value "^2.0.0" + to-object-path "^0.3.0" + union-value "^1.0.0" + unset-value "^1.0.0" + +call-bind@^1.0.0, call-bind@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" + integrity sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA== + dependencies: + function-bind "^1.1.1" + get-intrinsic "^1.0.2" + +call-me-maybe@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/call-me-maybe/-/call-me-maybe-1.0.1.tgz#26d208ea89e37b5cbde60250a15f031c16a4d66b" + integrity sha1-JtII6onje1y95gJQoV8DHBak1ms= + +callsites@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" + integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== + +camelcase-keys@^6.2.2: + version "6.2.2" + resolved "https://registry.yarnpkg.com/camelcase-keys/-/camelcase-keys-6.2.2.tgz#5e755d6ba51aa223ec7d3d52f25778210f9dc3c0" + integrity sha512-YrwaA0vEKazPBkn0ipTiMpSajYDSe+KjQfrjhcBMxJt/znbvlHd8Pw/Vamaz5EB4Wfhs3SUR3Z9mwRu/P3s3Yg== + dependencies: + camelcase "^5.3.1" + map-obj "^4.0.0" + quick-lru "^4.0.1" + +camelcase-keys@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/camelcase-keys/-/camelcase-keys-7.0.0.tgz#40fcbe171f7432888369d0c871df7cfa5ce4f788" + integrity sha512-qlQlECgDl5Ev+gkvONaiD4X4TF2gyZKuLBvzx0zLo2UwAxmz3hJP/841aaMHTeH1T7v5HRwoRq91daulXoYWvg== + dependencies: + camelcase "^6.2.0" + map-obj "^4.1.0" + quick-lru "^5.1.1" + type-fest "^1.2.1" + +camelcase@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" + integrity sha1-MvxLn82vhF/N9+c7uXysImHwqwo= + +camelcase@^5.0.0, camelcase@^5.3.1: + version "5.3.1" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320" + integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg== + +camelcase@^6.2.0: + version "6.2.0" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.2.0.tgz#924af881c9d525ac9d87f40d964e5cea982a1809" + integrity sha512-c7wVvbw3f37nuobQNtgsgG9POC9qMbNuMQmTCqZv23b6MIz0fcYpBiOlv9gEN/hdLdnZTDQhg6e9Dq5M1vKvfg== + +caniuse-lite@^1.0.30001219: + version "1.0.30001246" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001246.tgz#fe17d9919f87124d6bb416ef7b325356d69dc76c" + integrity sha512-Tc+ff0Co/nFNbLOrziBXmMVtpt9S2c2Y+Z9Nk9Khj09J+0zR9ejvIW5qkZAErCbOrVODCx/MN+GpB5FNBs5GFA== + +caseless@~0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" + integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw= + +chalk@2.x, chalk@^2.0.0, chalk@^2.4.1, chalk@^2.4.2: + version "2.4.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" + integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + +chalk@^4.0.0, chalk@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.1.tgz#c80b3fab28bf6371e6863325eee67e618b77e6ad" + integrity sha512-diHzdDKxcU+bAsUboHLPEDQiw0qEe0qd7SYUn3HgcFlWgbDcfLGswOHYeGrHKzG9z6UYf01d9VFMfZxPM1xZSg== + dependencies: + ansi-styles "^4.1.0" + supports-color "^7.1.0" + +char-regex@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf" + integrity sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw== + +chardet@^0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" + integrity sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA== + +chokidar@^2.0.0: + version "2.1.8" + resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917" + integrity sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg== + dependencies: + anymatch "^2.0.0" + async-each "^1.0.1" + braces "^2.3.2" + glob-parent "^3.1.0" + inherits "^2.0.3" + is-binary-path "^1.0.0" + is-glob "^4.0.0" + normalize-path "^3.0.0" + path-is-absolute "^1.0.0" + readdirp "^2.2.1" + upath "^1.1.1" + optionalDependencies: + fsevents "^1.2.7" + +chownr@^1.1.1: + version "1.1.4" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" + integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== + +chownr@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-2.0.0.tgz#15bfbe53d2eab4cf70f18a8cd68ebe5b3cb1dece" + integrity sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ== + +ci-info@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46" + integrity sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ== + +ci-info@^3.1.1: + version "3.2.0" + resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.2.0.tgz#2876cb948a498797b5236f0095bc057d0dca38b6" + integrity sha512-dVqRX7fLUm8J6FgHJ418XuIgDLZDkYcDFTeL6TA2gt5WlIZUQrrH6EZrNClwT/H0FateUsZkGIOPRrLbP+PR9A== + +cjs-module-lexer@^1.0.0: + version "1.2.2" + resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-1.2.2.tgz#9f84ba3244a512f3a54e5277e8eef4c489864e40" + integrity sha512-cOU9usZw8/dXIXKtwa8pM0OTJQuJkxMN6w30csNRUerHfeQ5R6U3kkU/FtJeIf3M202OHfY2U8ccInBG7/xogA== + +class-utils@^0.3.5: + version "0.3.6" + resolved "https://registry.yarnpkg.com/class-utils/-/class-utils-0.3.6.tgz#f93369ae8b9a7ce02fd41faad0ca83033190c463" + integrity sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg== + dependencies: + arr-union "^3.1.0" + define-property "^0.2.5" + isobject "^3.0.0" + static-extend "^0.1.1" + +clean-stack@^2.0.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" + integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== + +cli-cursor@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-3.1.0.tgz#264305a7ae490d1d03bf0c9ba7c925d1753af307" + integrity sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw== + dependencies: + restore-cursor "^3.1.0" + +cli-width@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-3.0.0.tgz#a2f48437a2caa9a22436e794bf071ec9e61cedf6" + integrity sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw== + +cliui@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/cliui/-/cliui-3.2.0.tgz#120601537a916d29940f934da3b48d585a39213d" + integrity sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0= + dependencies: + string-width "^1.0.1" + strip-ansi "^3.0.1" + wrap-ansi "^2.0.0" + +cliui@^7.0.2: + version "7.0.4" + resolved "https://registry.yarnpkg.com/cliui/-/cliui-7.0.4.tgz#a0265ee655476fc807aea9df3df8df7783808b4f" + integrity sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ== + dependencies: + string-width "^4.2.0" + strip-ansi "^6.0.0" + wrap-ansi "^7.0.0" + +clone-buffer@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/clone-buffer/-/clone-buffer-1.0.0.tgz#e3e25b207ac4e701af721e2cb5a16792cac3dc58" + integrity sha1-4+JbIHrE5wGvch4staFnksrD3Fg= + +clone-deep@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/clone-deep/-/clone-deep-4.0.1.tgz#c19fd9bdbbf85942b4fd979c84dcf7d5f07c2387" + integrity sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ== + dependencies: + is-plain-object "^2.0.4" + kind-of "^6.0.2" + shallow-clone "^3.0.0" + +clone-stats@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/clone-stats/-/clone-stats-1.0.0.tgz#b3782dff8bb5474e18b9b6bf0fdfe782f8777680" + integrity sha1-s3gt/4u1R04Yuba/D9/ngvh3doA= + +clone@^1.0.2: + version "1.0.4" + resolved "https://registry.yarnpkg.com/clone/-/clone-1.0.4.tgz#da309cc263df15994c688ca902179ca3c7cd7c7e" + integrity sha1-2jCcwmPfFZlMaIypAheco8fNfH4= + +clone@^2.1.1: + version "2.1.2" + resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f" + integrity sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18= + +cloneable-readable@^1.0.0: + version "1.1.3" + resolved "https://registry.yarnpkg.com/cloneable-readable/-/cloneable-readable-1.1.3.tgz#120a00cb053bfb63a222e709f9683ea2e11d8cec" + integrity sha512-2EF8zTQOxYq70Y4XKtorQupqF0m49MBz2/yf5Bj+MHjvpG3Hy7sImifnqD6UA+TKYxeSV+u6qqQPawN5UvnpKQ== + dependencies: + inherits "^2.0.1" + process-nextick-args "^2.0.0" + readable-stream "^2.3.5" + +cmd-shim@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/cmd-shim/-/cmd-shim-4.1.0.tgz#b3a904a6743e9fede4148c6f3800bf2a08135bdd" + integrity sha512-lb9L7EM4I/ZRVuljLPEtUJOP+xiQVknZ4ZMpMgEp4JzNldPb27HU03hi6K1/6CoIuit/Zm/LQXySErFeXxDprw== + dependencies: + mkdirp-infer-owner "^2.0.0" + +co@^4.6.0: + version "4.6.0" + resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" + integrity sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ= + +code-point-at@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" + integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c= + +collect-v8-coverage@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/collect-v8-coverage/-/collect-v8-coverage-1.0.1.tgz#cc2c8e94fc18bbdffe64d6534570c8a673b27f59" + integrity sha512-iBPtljfCNcTKNAto0KEtDfZ3qzjJvqE3aTGZsbhjSBlorqpXJlaWWtPO35D+ZImoC3KWejX64o+yPGxhWSTzfg== + +collection-map@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/collection-map/-/collection-map-1.0.0.tgz#aea0f06f8d26c780c2b75494385544b2255af18c" + integrity sha1-rqDwb40mx4DCt1SUOFVEsiVa8Yw= + dependencies: + arr-map "^2.0.2" + for-own "^1.0.0" + make-iterator "^1.0.0" + +collection-visit@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/collection-visit/-/collection-visit-1.0.0.tgz#4bc0373c164bc3291b4d368c829cf1a80a59dca0" + integrity sha1-S8A3PBZLwykbTTaMgpzxqApZ3KA= + dependencies: + map-visit "^1.0.0" + object-visit "^1.0.0" + +color-convert@^1.9.0: + version "1.9.3" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" + integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== + dependencies: + color-name "1.1.3" + +color-convert@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" + integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== + dependencies: + color-name "~1.1.4" + +color-name@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" + integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= + +color-name@~1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" + integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== + +color-support@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/color-support/-/color-support-1.1.3.tgz#93834379a1cc9a0c61f82f52f0d04322251bd5a2" + integrity sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg== + +colorette@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.2.2.tgz#cbcc79d5e99caea2dbf10eb3a26fd8b3e6acfa94" + integrity sha512-MKGMzyfeuutC/ZJ1cba9NqcNpfeqMUcYmyF1ZFY6/Cn7CNSAKx6a+s48sqLqyAiZuaP2TcqMhoo+dlwFnVxT9w== + +columnify@^1.5.4: + version "1.5.4" + resolved "https://registry.yarnpkg.com/columnify/-/columnify-1.5.4.tgz#4737ddf1c7b69a8a7c340570782e947eec8e78bb" + integrity sha1-Rzfd8ce2mop8NAVweC6UfuyOeLs= + dependencies: + strip-ansi "^3.0.0" + wcwidth "^1.0.0" + +combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: + version "1.0.8" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== + dependencies: + delayed-stream "~1.0.0" + +command-line-args@5.1.3: + version "5.1.3" + resolved "https://registry.yarnpkg.com/command-line-args/-/command-line-args-5.1.3.tgz#1e57d2816f28804073bb5e75cd24e02e2aa321e7" + integrity sha512-a5tF6mjqRSOBswBwdMkKY47JQ464Dkg9Pcwbxwo9wxRhKWZjtBktmBASllk3AMJ7qBuWgsAGtVa7b2/+EsymOQ== + dependencies: + array-back "^3.1.0" + find-replace "^3.0.0" + lodash.camelcase "^4.3.0" + typical "^4.0.0" + +command-line-usage@6.1.1: + version "6.1.1" + resolved "https://registry.yarnpkg.com/command-line-usage/-/command-line-usage-6.1.1.tgz#c908e28686108917758a49f45efb4f02f76bc03f" + integrity sha512-F59pEuAR9o1SF/bD0dQBDluhpT4jJQNWUHEuVBqpDmCUo6gPjCi+m9fCWnWZVR/oG6cMTUms4h+3NPl74wGXvA== + dependencies: + array-back "^4.0.1" + chalk "^2.4.2" + table-layout "^1.0.1" + typical "^5.2.0" + +commander@^6.1.0: + version "6.2.1" + resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c" + integrity sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA== + +compare-func@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/compare-func/-/compare-func-2.0.0.tgz#fb65e75edbddfd2e568554e8b5b05fff7a51fcb3" + integrity sha512-zHig5N+tPWARooBnb0Zx1MFcdfpyJrfTJ3Y5L+IFvUm8rM74hHz66z0gw0x4tijh5CorKkKUCnW82R2vmpeCRA== + dependencies: + array-ify "^1.0.0" + dot-prop "^5.1.0" + +component-emitter@^1.2.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.3.0.tgz#16e4070fba8ae29b679f2215853ee181ab2eabc0" + integrity sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg== + +concat-map@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" + integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= + +concat-stream@^1.6.0: + version "1.6.2" + resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.6.2.tgz#904bdf194cd3122fc675c77fc4ac3d4ff0fd1a34" + integrity sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw== + dependencies: + buffer-from "^1.0.0" + inherits "^2.0.3" + readable-stream "^2.2.2" + typedarray "^0.0.6" + +concat-stream@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-2.0.0.tgz#414cf5af790a48c60ab9be4527d56d5e41133cb1" + integrity sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A== + dependencies: + buffer-from "^1.0.0" + inherits "^2.0.3" + readable-stream "^3.0.2" + typedarray "^0.0.6" + +config-chain@^1.1.12: + version "1.1.13" + resolved "https://registry.yarnpkg.com/config-chain/-/config-chain-1.1.13.tgz#fad0795aa6a6cdaff9ed1b68e9dff94372c232f4" + integrity sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ== + dependencies: + ini "^1.3.4" + proto-list "~1.2.1" + +console-control-strings@^1.0.0, console-control-strings@~1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" + integrity sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4= + +conventional-changelog-angular@^5.0.12: + version "5.0.12" + resolved "https://registry.yarnpkg.com/conventional-changelog-angular/-/conventional-changelog-angular-5.0.12.tgz#c979b8b921cbfe26402eb3da5bbfda02d865a2b9" + integrity sha512-5GLsbnkR/7A89RyHLvvoExbiGbd9xKdKqDTrArnPbOqBqG/2wIosu0fHwpeIRI8Tl94MhVNBXcLJZl92ZQ5USw== + dependencies: + compare-func "^2.0.0" + q "^1.5.1" + +conventional-changelog-core@^4.2.2: + version "4.2.3" + resolved "https://registry.yarnpkg.com/conventional-changelog-core/-/conventional-changelog-core-4.2.3.tgz#ce44d4bbba4032e3dc14c00fcd5b53fc00b66433" + integrity sha512-MwnZjIoMRL3jtPH5GywVNqetGILC7g6RQFvdb8LRU/fA/338JbeWAku3PZ8yQ+mtVRViiISqJlb0sOz0htBZig== + dependencies: + add-stream "^1.0.0" + conventional-changelog-writer "^5.0.0" + conventional-commits-parser "^3.2.0" + dateformat "^3.0.0" + get-pkg-repo "^4.0.0" + git-raw-commits "^2.0.8" + git-remote-origin-url "^2.0.0" + git-semver-tags "^4.1.1" + lodash "^4.17.15" + normalize-package-data "^3.0.0" + q "^1.5.1" + read-pkg "^3.0.0" + read-pkg-up "^3.0.0" + through2 "^4.0.0" + +conventional-changelog-preset-loader@^2.3.4: + version "2.3.4" + resolved "https://registry.yarnpkg.com/conventional-changelog-preset-loader/-/conventional-changelog-preset-loader-2.3.4.tgz#14a855abbffd59027fd602581f1f34d9862ea44c" + integrity sha512-GEKRWkrSAZeTq5+YjUZOYxdHq+ci4dNwHvpaBC3+ENalzFWuCWa9EZXSuZBpkr72sMdKB+1fyDV4takK1Lf58g== + +conventional-changelog-writer@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/conventional-changelog-writer/-/conventional-changelog-writer-5.0.0.tgz#c4042f3f1542f2f41d7d2e0d6cad23aba8df8eec" + integrity sha512-HnDh9QHLNWfL6E1uHz6krZEQOgm8hN7z/m7tT16xwd802fwgMN0Wqd7AQYVkhpsjDUx/99oo+nGgvKF657XP5g== + dependencies: + conventional-commits-filter "^2.0.7" + dateformat "^3.0.0" + handlebars "^4.7.6" + json-stringify-safe "^5.0.1" + lodash "^4.17.15" + meow "^8.0.0" + semver "^6.0.0" + split "^1.0.0" + through2 "^4.0.0" + +conventional-commits-filter@^2.0.7: + version "2.0.7" + resolved "https://registry.yarnpkg.com/conventional-commits-filter/-/conventional-commits-filter-2.0.7.tgz#f8d9b4f182fce00c9af7139da49365b136c8a0b3" + integrity sha512-ASS9SamOP4TbCClsRHxIHXRfcGCnIoQqkvAzCSbZzTFLfcTqJVugB0agRgsEELsqaeWgsXv513eS116wnlSSPA== + dependencies: + lodash.ismatch "^4.4.0" + modify-values "^1.0.0" + +conventional-commits-parser@^3.2.0: + version "3.2.1" + resolved "https://registry.yarnpkg.com/conventional-commits-parser/-/conventional-commits-parser-3.2.1.tgz#ba44f0b3b6588da2ee9fd8da508ebff50d116ce2" + integrity sha512-OG9kQtmMZBJD/32NEw5IhN5+HnBqVjy03eC+I71I0oQRFA5rOgA4OtPOYG7mz1GkCfCNxn3gKIX8EiHJYuf1cA== + dependencies: + JSONStream "^1.0.4" + is-text-path "^1.0.1" + lodash "^4.17.15" + meow "^8.0.0" + split2 "^3.0.0" + through2 "^4.0.0" + trim-off-newlines "^1.0.0" + +conventional-recommended-bump@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/conventional-recommended-bump/-/conventional-recommended-bump-6.1.0.tgz#cfa623285d1de554012f2ffde70d9c8a22231f55" + integrity sha512-uiApbSiNGM/kkdL9GTOLAqC4hbptObFo4wW2QRyHsKciGAfQuLU1ShZ1BIVI/+K2BE/W1AWYQMCXAsv4dyKPaw== + dependencies: + concat-stream "^2.0.0" + conventional-changelog-preset-loader "^2.3.4" + conventional-commits-filter "^2.0.7" + conventional-commits-parser "^3.2.0" + git-raw-commits "^2.0.8" + git-semver-tags "^4.1.1" + meow "^8.0.0" + q "^1.5.1" + +convert-source-map@^1.0.0, convert-source-map@^1.4.0, convert-source-map@^1.5.0, convert-source-map@^1.6.0, convert-source-map@^1.7.0: + version "1.8.0" + resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.8.0.tgz#f3373c32d21b4d780dd8004514684fb791ca4369" + integrity sha512-+OQdjP49zViI/6i7nIJpA8rAl4sV/JdPfU9nZs3VqOwGIgizICvuN2ru6fMd+4llL0tar18UYJXfZ/TWtmhUjA== + dependencies: + safe-buffer "~5.1.1" + +copy-descriptor@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/copy-descriptor/-/copy-descriptor-0.1.1.tgz#676f6eb3c39997c2ee1ac3a924fd6124748f578d" + integrity sha1-Z29us8OZl8LuGsOpJP1hJHSPV40= + +copy-props@^2.0.1: + version "2.0.5" + resolved "https://registry.yarnpkg.com/copy-props/-/copy-props-2.0.5.tgz#03cf9ae328d4ebb36f8f1d804448a6af9ee3f2d2" + integrity sha512-XBlx8HSqrT0ObQwmSzM7WE5k8FxTV75h1DX1Z3n6NhQ/UYYAvInWYmG06vFt7hQZArE2fuO62aihiWIVQwh1sw== + dependencies: + each-props "^1.3.2" + is-plain-object "^5.0.0" + +core-util-is@1.0.2, core-util-is@~1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" + integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= + +cosmiconfig@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-7.0.0.tgz#ef9b44d773959cae63ddecd122de23853b60f8d3" + integrity sha512-pondGvTuVYDk++upghXJabWzL6Kxu6f26ljFw64Swq9v6sQPUL3EUlVDV56diOjpCayKihL6hVe8exIACU4XcA== + dependencies: + "@types/parse-json" "^4.0.0" + import-fresh "^3.2.1" + parse-json "^5.0.0" + path-type "^4.0.0" + yaml "^1.10.0" + +cp-file@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/cp-file/-/cp-file-7.0.0.tgz#b9454cfd07fe3b974ab9ea0e5f29655791a9b8cd" + integrity sha512-0Cbj7gyvFVApzpK/uhCtQ/9kE9UnYpxMzaq5nQQC/Dh4iaj5fxp7iEFIullrYwzj8nf0qnsI1Qsx34hAeAebvw== + dependencies: + graceful-fs "^4.1.2" + make-dir "^3.0.0" + nested-error-stacks "^2.0.0" + p-event "^4.1.0" + +cpy@8.1.2: + version "8.1.2" + resolved "https://registry.yarnpkg.com/cpy/-/cpy-8.1.2.tgz#e339ea54797ad23f8e3919a5cffd37bfc3f25935" + integrity sha512-dmC4mUesv0OYH2kNFEidtf/skUwv4zePmGeepjyyJ0qTo5+8KhA1o99oIAwVVLzQMAeDJml74d6wPPKb6EZUTg== + dependencies: + arrify "^2.0.1" + cp-file "^7.0.0" + globby "^9.2.0" + has-glob "^1.0.0" + junk "^3.1.0" + nested-error-stacks "^2.1.0" + p-all "^2.1.0" + p-filter "^2.1.0" + p-map "^3.0.0" + +create-require@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" + integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== + +cross-env@7.0.3: + version "7.0.3" + resolved "https://registry.yarnpkg.com/cross-env/-/cross-env-7.0.3.tgz#865264b29677dc015ba8418918965dd232fc54cf" + integrity sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw== + dependencies: + cross-spawn "^7.0.1" + +cross-spawn@^6.0.5: + version "6.0.5" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4" + integrity sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ== + dependencies: + nice-try "^1.0.4" + path-key "^2.0.1" + semver "^5.5.0" + shebang-command "^1.2.0" + which "^1.2.9" + +cross-spawn@^7.0.1, cross-spawn@^7.0.2, cross-spawn@^7.0.3: + version "7.0.3" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6" + integrity sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w== + dependencies: + path-key "^3.1.0" + shebang-command "^2.0.0" + which "^2.0.1" + +css@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/css/-/css-3.0.0.tgz#4447a4d58fdd03367c516ca9f64ae365cee4aa5d" + integrity sha512-DG9pFfwOrzc+hawpmqX/dHYHJG+Bsdb0klhyi1sDneOgGOXy9wQIC8hzyVp1e4NRYDBdxcylvywPkkXCHAzTyQ== + dependencies: + inherits "^2.0.4" + source-map "^0.6.1" + source-map-resolve "^0.6.0" + +cssom@^0.4.4: + version "0.4.4" + resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.4.4.tgz#5a66cf93d2d0b661d80bf6a44fb65f5c2e4e0a10" + integrity sha512-p3pvU7r1MyyqbTk+WbNJIgJjG2VmTIaB10rI93LzVPrmDJKkzKYMtxxyAvQXR/NS6otuzveI7+7BBq3SjBS2mw== + +cssom@~0.3.6: + version "0.3.8" + resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.3.8.tgz#9f1276f5b2b463f2114d3f2c75250af8c1a36f4a" + integrity sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg== + +cssstyle@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/cssstyle/-/cssstyle-2.3.0.tgz#ff665a0ddbdc31864b09647f34163443d90b0852" + integrity sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A== + dependencies: + cssom "~0.3.6" + +d@1, d@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/d/-/d-1.0.1.tgz#8698095372d58dbee346ffd0c7093f99f8f9eb5a" + integrity sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA== + dependencies: + es5-ext "^0.10.50" + type "^1.0.1" + +dargs@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/dargs/-/dargs-7.0.0.tgz#04015c41de0bcb69ec84050f3d9be0caf8d6d5cc" + integrity sha512-2iy1EkLdlBzQGvbweYRFxmFath8+K7+AKB0TlhHWkNuH+TmovaMH/Wp7V7R4u7f4SnX3OgLsU9t1NI9ioDnUpg== + +dashdash@^1.12.0: + version "1.14.1" + resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" + integrity sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA= + dependencies: + assert-plus "^1.0.0" + +data-urls@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-2.0.0.tgz#156485a72963a970f5d5821aaf642bef2bf2db9b" + integrity sha512-X5eWTSXO/BJmpdIKCRuKUgSCgAN0OwliVK3yPKbwIWU1Tdw5BRajxlzMidvh+gwko9AfQ9zIj52pzF91Q3YAvQ== + dependencies: + abab "^2.0.3" + whatwg-mimetype "^2.3.0" + whatwg-url "^8.0.0" + +dateformat@^3.0.0: + version "3.0.3" + resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-3.0.3.tgz#a6e37499a4d9a9cf85ef5872044d62901c9889ae" + integrity sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q== + +debug-fabulous@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/debug-fabulous/-/debug-fabulous-1.1.0.tgz#af8a08632465224ef4174a9f06308c3c2a1ebc8e" + integrity sha512-GZqvGIgKNlUnHUPQhepnUZFIMoi3dgZKQBzKDeL2g7oJF9SNAji/AAu36dusFUas0O+pae74lNeoIPHqXWDkLg== + dependencies: + debug "3.X" + memoizee "0.4.X" + object-assign "4.X" + +debug@3.X: + version "3.2.7" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" + integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== + dependencies: + ms "^2.1.1" + +debug@4, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1: + version "4.3.2" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b" + integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw== + dependencies: + ms "2.1.2" + +debug@^2.2.0, debug@^2.3.3: + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" + integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== + dependencies: + ms "2.0.0" + +debuglog@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492" + integrity sha1-qiT/uaw9+aI1GDfPstJ5NgzXhJI= + +decamelize-keys@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/decamelize-keys/-/decamelize-keys-1.1.0.tgz#d171a87933252807eb3cb61dc1c1445d078df2d9" + integrity sha1-0XGoeTMlKAfrPLYdwcFEXQeN8tk= + dependencies: + decamelize "^1.1.0" + map-obj "^1.0.0" + +decamelize@^1.1.0, decamelize@^1.1.1, decamelize@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" + integrity sha1-9lNNFRSCabIDUue+4m9QH5oZEpA= + +decamelize@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-5.0.0.tgz#88358157b010ef133febfd27c18994bd80c6215b" + integrity sha512-U75DcT5hrio3KNtvdULAWnLiAPbFUC4191ldxMmj4FA/mRuBnmDwU0boNfPyFRhnan+Jm+haLeSn3P0afcBn4w== + +decimal.js@^10.2.1: + version "10.3.1" + resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.3.1.tgz#d8c3a444a9c6774ba60ca6ad7261c3a94fd5e783" + integrity sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ== + +decode-uri-component@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545" + integrity sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU= + +dedent@^0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/dedent/-/dedent-0.7.0.tgz#2495ddbaf6eb874abb0e1be9df22d2e5a544326c" + integrity sha1-JJXduvbrh0q7Dhvp3yLS5aVEMmw= + +deep-extend@~0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac" + integrity sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA== + +deep-is@^0.1.3, deep-is@~0.1.3: + version "0.1.3" + resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" + integrity sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ= + +deepmerge@^4.2.2: + version "4.2.2" + resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.2.2.tgz#44d2ea3679b8f4d4ffba33f03d865fc1e7bf4955" + integrity sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg== + +default-compare@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/default-compare/-/default-compare-1.0.0.tgz#cb61131844ad84d84788fb68fd01681ca7781a2f" + integrity sha512-QWfXlM0EkAbqOCbD/6HjdwT19j7WCkMyiRhWilc4H9/5h/RzTF9gv5LYh1+CmDV5d1rki6KAWLtQale0xt20eQ== + dependencies: + kind-of "^5.0.2" + +default-resolution@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/default-resolution/-/default-resolution-2.0.0.tgz#bcb82baa72ad79b426a76732f1a81ad6df26d684" + integrity sha1-vLgrqnKtebQmp2cy8aga1t8m1oQ= + +defaults@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/defaults/-/defaults-1.0.3.tgz#c656051e9817d9ff08ed881477f3fe4019f3ef7d" + integrity sha1-xlYFHpgX2f8I7YgUd/P+QBnz730= + dependencies: + clone "^1.0.2" + +define-properties@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.3.tgz#cf88da6cbee26fe6db7094f61d870cbd84cee9f1" + integrity sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ== + dependencies: + object-keys "^1.0.12" + +define-property@^0.2.5: + version "0.2.5" + resolved "https://registry.yarnpkg.com/define-property/-/define-property-0.2.5.tgz#c35b1ef918ec3c990f9a5bc57be04aacec5c8116" + integrity sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY= + dependencies: + is-descriptor "^0.1.0" + +define-property@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/define-property/-/define-property-1.0.0.tgz#769ebaaf3f4a63aad3af9e8d304c9bbe79bfb0e6" + integrity sha1-dp66rz9KY6rTr56NMEybvnm/sOY= + dependencies: + is-descriptor "^1.0.0" + +define-property@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/define-property/-/define-property-2.0.2.tgz#d459689e8d654ba77e02a817f8710d702cb16e9d" + integrity sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ== + dependencies: + is-descriptor "^1.0.2" + isobject "^3.0.1" + +del-cli@4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/del-cli/-/del-cli-4.0.1.tgz#2303ccaa45708ee8c6211568344cf87336abf30a" + integrity sha512-KtR/6cBfZkGDAP2NA7z+bP4p1OMob3wjN9mq13+SWvExx6jT9gFWfLgXEeX8J2B47OKeNCq9yTONmtryQ+m+6g== + dependencies: + del "^6.0.0" + meow "^10.1.0" + +del@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/del/-/del-6.0.0.tgz#0b40d0332cea743f1614f818be4feb717714c952" + integrity sha512-1shh9DQ23L16oXSZKB2JxpL7iMy2E0S9d517ptA1P8iw0alkPtQcrKH7ru31rYtKwF499HkTu+DRzq3TCKDFRQ== + dependencies: + globby "^11.0.1" + graceful-fs "^4.2.4" + is-glob "^4.0.1" + is-path-cwd "^2.2.0" + is-path-inside "^3.0.2" + p-map "^4.0.0" + rimraf "^3.0.2" + slash "^3.0.0" + +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= + +delegates@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" + integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o= + +depd@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" + integrity sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak= + +deprecation@^2.0.0, deprecation@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/deprecation/-/deprecation-2.3.1.tgz#6368cbdb40abf3373b525ac87e4a260c3a700919" + integrity sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ== + +detect-file@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/detect-file/-/detect-file-1.0.0.tgz#f0d66d03672a825cb1b73bdb3fe62310c8e552b7" + integrity sha1-8NZtA2cqglyxtzvbP+YjEMjlUrc= + +detect-indent@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-5.0.0.tgz#3871cc0a6a002e8c3e5b3cf7f336264675f06b9d" + integrity sha1-OHHMCmoALow+Wzz38zYmRnXwa50= + +detect-indent@^6.0.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-6.1.0.tgz#592485ebbbf6b3b1ab2be175c8393d04ca0d57e6" + integrity sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA== + +detect-newline@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-2.1.0.tgz#f41f1c10be4b00e87b5f13da680759f2c5bfd3e2" + integrity sha1-9B8cEL5LAOh7XxPaaAdZ8sW/0+I= + +detect-newline@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" + integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA== + +dezalgo@^1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/dezalgo/-/dezalgo-1.0.3.tgz#7f742de066fc748bc8db820569dddce49bf0d456" + integrity sha1-f3Qt4Gb8dIvI24IFad3c5Jvw1FY= + dependencies: + asap "^2.0.0" + wrappy "1" + +diff-sequences@^26.6.2: + version "26.6.2" + resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-26.6.2.tgz#48ba99157de1923412eed41db6b6d4aa9ca7c0b1" + integrity sha512-Mv/TDa3nZ9sbc5soK+OoA74BsS3mL37yixCvUAQkiuA4Wz6YtwP/K47n2rv2ovzHZvoiQeA5FTQOschKkEwB0Q== + +diff-sequences@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-27.0.6.tgz#3305cb2e55a033924054695cc66019fd7f8e5723" + integrity sha512-ag6wfpBFyNXZ0p8pcuIDS//D8H062ZQJ3fzYxjpmeKjnz8W4pekL3AI8VohmyZmsWW2PWaHgjsmqR6L13101VQ== + +diff@^4.0.1: + version "4.0.2" + resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" + integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== + +dir-glob@^2.2.2: + version "2.2.2" + resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-2.2.2.tgz#fa09f0694153c8918b18ba0deafae94769fc50c4" + integrity sha512-f9LBi5QWzIW3I6e//uxZoLBlUt9kcp66qo0sSCxL6YZKc75R1c4MFCoe/LaZiBGmgujvQdxc5Bn3QhfyvK5Hsw== + dependencies: + path-type "^3.0.0" + +dir-glob@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" + integrity sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA== + dependencies: + path-type "^4.0.0" + +doctrine@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-3.0.0.tgz#addebead72a6574db783639dc87a121773973961" + integrity sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w== + dependencies: + esutils "^2.0.2" + +domexception@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/domexception/-/domexception-2.0.1.tgz#fb44aefba793e1574b0af6aed2801d057529f304" + integrity sha512-yxJ2mFy/sibVQlu5qHjOkf9J3K6zgmCxgJ94u2EdvDOV09H+32LtRswEcUsmUWN72pVLOEnTSRaIVVzVQgS0dg== + dependencies: + webidl-conversions "^5.0.0" + +dot-prop@^5.1.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-5.3.0.tgz#90ccce708cd9cd82cc4dc8c3ddd9abdd55b20e88" + integrity sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q== + dependencies: + is-obj "^2.0.0" + +dot-prop@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-6.0.1.tgz#fc26b3cf142b9e59b74dbd39ed66ce620c681083" + integrity sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA== + dependencies: + is-obj "^2.0.0" + +duplexer@^0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.2.tgz#3abe43aef3835f8ae077d136ddce0f276b0400e6" + integrity sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg== + +duplexify@^3.6.0: + version "3.7.1" + resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-3.7.1.tgz#2a4df5317f6ccfd91f86d6fd25d8d8a103b88309" + integrity sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g== + dependencies: + end-of-stream "^1.0.0" + inherits "^2.0.1" + readable-stream "^2.0.0" + stream-shift "^1.0.0" + +each-props@^1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/each-props/-/each-props-1.3.2.tgz#ea45a414d16dd5cfa419b1a81720d5ca06892333" + integrity sha512-vV0Hem3zAGkJAyU7JSjixeU66rwdynTAa1vofCrSA5fEln+m67Az9CcnkVD776/fsN/UjIWmBDoNRS6t6G9RfA== + dependencies: + is-plain-object "^2.0.1" + object.defaults "^1.1.0" + +ecc-jsbn@~0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" + integrity sha1-OoOpBOVDUyh4dMVkt1SThoSamMk= + dependencies: + jsbn "~0.1.0" + safer-buffer "^2.1.0" + +electron-to-chromium@^1.3.723: + version "1.3.782" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.782.tgz#522740fe6b4b5255ca754c68d9c406a17b0998e2" + integrity sha512-6AI2se1NqWA1SBf/tlD6tQD/6ZOt+yAhqmrTlh4XZw4/g0Mt3p6JhTQPZxRPxPZiOg0o7ss1EBP/CpYejfnoIA== + +emittery@^0.8.1: + version "0.8.1" + resolved "https://registry.yarnpkg.com/emittery/-/emittery-0.8.1.tgz#bb23cc86d03b30aa75a7f734819dee2e1ba70860" + integrity sha512-uDfvUjVrfGJJhymx/kz6prltenw1u7WrCg1oa94zYY8xxVpLLUu045LAT0dhDZdXG58/EpPL/5kA180fQ/qudg== + +emoji-regex@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" + integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== + +encoding@^0.1.12: + version "0.1.13" + resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.13.tgz#56574afdd791f54a8e9b2785c0582a2d26210fa9" + integrity sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A== + dependencies: + iconv-lite "^0.6.2" + +end-of-stream@^1.0.0, end-of-stream@^1.1.0: + version "1.4.4" + resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" + integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== + dependencies: + once "^1.4.0" + +enquirer@^2.3.5: + version "2.3.6" + resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d" + integrity sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg== + dependencies: + ansi-colors "^4.1.1" + +env-paths@^2.2.0: + version "2.2.1" + resolved "https://registry.yarnpkg.com/env-paths/-/env-paths-2.2.1.tgz#420399d416ce1fbe9bc0a07c62fa68d67fd0f8f2" + integrity sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A== + +envinfo@^7.7.4: + version "7.8.1" + resolved "https://registry.yarnpkg.com/envinfo/-/envinfo-7.8.1.tgz#06377e3e5f4d379fea7ac592d5ad8927e0c4d475" + integrity sha512-/o+BXHmB7ocbHEAs6F2EnG0ogybVVUdkRunTT2glZU9XAaGmhqskrvKwqXuDfNjEO0LZKWdejEEpnq8aM0tOaw== + +err-code@^2.0.2: + version "2.0.3" + resolved "https://registry.yarnpkg.com/err-code/-/err-code-2.0.3.tgz#23c2f3b756ffdfc608d30e27c9a941024807e7f9" + integrity sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA== + +error-ex@^1.2.0, error-ex@^1.3.1: + version "1.3.2" + resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" + integrity sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g== + dependencies: + is-arrayish "^0.2.1" + +es-abstract@^1.18.0-next.2: + version "1.18.3" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.18.3.tgz#25c4c3380a27aa203c44b2b685bba94da31b63e0" + integrity sha512-nQIr12dxV7SSxE6r6f1l3DtAeEYdsGpps13dR0TwJg1S8gyp4ZPgy3FZcHBgbiQqnoqSTb+oC+kO4UQ0C/J8vw== + dependencies: + call-bind "^1.0.2" + es-to-primitive "^1.2.1" + function-bind "^1.1.1" + get-intrinsic "^1.1.1" + has "^1.0.3" + has-symbols "^1.0.2" + is-callable "^1.2.3" + is-negative-zero "^2.0.1" + is-regex "^1.1.3" + is-string "^1.0.6" + object-inspect "^1.10.3" + object-keys "^1.1.1" + object.assign "^4.1.2" + string.prototype.trimend "^1.0.4" + string.prototype.trimstart "^1.0.4" + unbox-primitive "^1.0.1" + +es-to-primitive@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.2.1.tgz#e55cd4c9cdc188bcefb03b366c736323fc5c898a" + integrity sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA== + dependencies: + is-callable "^1.1.4" + is-date-object "^1.0.1" + is-symbol "^1.0.2" + +es5-ext@^0.10.35, es5-ext@^0.10.46, es5-ext@^0.10.50, es5-ext@^0.10.53, es5-ext@~0.10.14, es5-ext@~0.10.2, es5-ext@~0.10.46: + version "0.10.53" + resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.53.tgz#93c5a3acfdbef275220ad72644ad02ee18368de1" + integrity sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q== + dependencies: + es6-iterator "~2.0.3" + es6-symbol "~3.1.3" + next-tick "~1.0.0" + +es6-iterator@^2.0.1, es6-iterator@^2.0.3, es6-iterator@~2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/es6-iterator/-/es6-iterator-2.0.3.tgz#a7de889141a05a94b0854403b2d0a0fbfa98f3b7" + integrity sha1-p96IkUGgWpSwhUQDstCg+/qY87c= + dependencies: + d "1" + es5-ext "^0.10.35" + es6-symbol "^3.1.1" + +es6-symbol@^3.1.1, es6-symbol@~3.1.3: + version "3.1.3" + resolved "https://registry.yarnpkg.com/es6-symbol/-/es6-symbol-3.1.3.tgz#bad5d3c1bcdac28269f4cb331e431c78ac705d18" + integrity sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA== + dependencies: + d "^1.0.1" + ext "^1.1.2" + +es6-weak-map@^2.0.1, es6-weak-map@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/es6-weak-map/-/es6-weak-map-2.0.3.tgz#b6da1f16cc2cc0d9be43e6bdbfc5e7dfcdf31d53" + integrity sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA== + dependencies: + d "1" + es5-ext "^0.10.46" + es6-iterator "^2.0.3" + es6-symbol "^3.1.1" + +escalade@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" + integrity sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw== + +escape-string-regexp@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" + integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ= + +escape-string-regexp@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz#a30304e99daa32e23b2fd20f51babd07cffca344" + integrity sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w== + +escape-string-regexp@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34" + integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== + +escodegen@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-2.0.0.tgz#5e32b12833e8aa8fa35e1bf0befa89380484c7dd" + integrity sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw== + dependencies: + esprima "^4.0.1" + estraverse "^5.2.0" + esutils "^2.0.2" + optionator "^0.8.1" + optionalDependencies: + source-map "~0.6.1" + +eslint-plugin-jest@24.3.7: + version "24.3.7" + resolved "https://registry.yarnpkg.com/eslint-plugin-jest/-/eslint-plugin-jest-24.3.7.tgz#a4deaa9e88182b92533a9c25cc4f3c369d7f33eb" + integrity sha512-pXED2NA4q2M/5mxlN6GyuUXAFJndT0uosOkQCHaUED9pqgBPd89ZzpcZEU6c5HtZNahC00M36FkwLdDHMDqaHw== + dependencies: + "@typescript-eslint/experimental-utils" "^4.0.1" + +eslint-scope@^5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-5.1.1.tgz#e786e59a66cb92b3f6c1fb0d508aab174848f48c" + integrity sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw== + dependencies: + esrecurse "^4.3.0" + estraverse "^4.1.1" + +eslint-utils@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/eslint-utils/-/eslint-utils-2.1.0.tgz#d2de5e03424e707dc10c74068ddedae708741b27" + integrity sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg== + dependencies: + eslint-visitor-keys "^1.1.0" + +eslint-utils@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/eslint-utils/-/eslint-utils-3.0.0.tgz#8aebaface7345bb33559db0a1f13a1d2d48c3672" + integrity sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA== + dependencies: + eslint-visitor-keys "^2.0.0" + +eslint-visitor-keys@^1.1.0, eslint-visitor-keys@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz#30ebd1ef7c2fdff01c3a4f151044af25fab0523e" + integrity sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ== + +eslint-visitor-keys@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz#f65328259305927392c938ed44eb0a5c9b2bd303" + integrity sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw== + +eslint@7.31.0: + version "7.31.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-7.31.0.tgz#f972b539424bf2604907a970860732c5d99d3aca" + integrity sha512-vafgJpSh2ia8tnTkNUkwxGmnumgckLh5aAbLa1xRmIn9+owi8qBNGKL+B881kNKNTy7FFqTEkpNkUvmw0n6PkA== + dependencies: + "@babel/code-frame" "7.12.11" + "@eslint/eslintrc" "^0.4.3" + "@humanwhocodes/config-array" "^0.5.0" + ajv "^6.10.0" + chalk "^4.0.0" + cross-spawn "^7.0.2" + debug "^4.0.1" + doctrine "^3.0.0" + enquirer "^2.3.5" + escape-string-regexp "^4.0.0" + eslint-scope "^5.1.1" + eslint-utils "^2.1.0" + eslint-visitor-keys "^2.0.0" + espree "^7.3.1" + esquery "^1.4.0" + esutils "^2.0.2" + fast-deep-equal "^3.1.3" + file-entry-cache "^6.0.1" + functional-red-black-tree "^1.0.1" + glob-parent "^5.1.2" + globals "^13.6.0" + ignore "^4.0.6" + import-fresh "^3.0.0" + imurmurhash "^0.1.4" + is-glob "^4.0.0" + js-yaml "^3.13.1" + json-stable-stringify-without-jsonify "^1.0.1" + levn "^0.4.1" + lodash.merge "^4.6.2" + minimatch "^3.0.4" + natural-compare "^1.4.0" + optionator "^0.9.1" + progress "^2.0.0" + regexpp "^3.1.0" + semver "^7.2.1" + strip-ansi "^6.0.0" + strip-json-comments "^3.1.0" + table "^6.0.9" + text-table "^0.2.0" + v8-compile-cache "^2.0.3" + +"esm@https://github.com/jsg2021/esm/releases/download/v3.x.x-pr883/esm-3.x.x-pr883.tgz": + version "3.2.25" + resolved "https://github.com/jsg2021/esm/releases/download/v3.x.x-pr883/esm-3.x.x-pr883.tgz#c463cfa4e14aceea6b7cd7e669ef90de072ea60a" + +espree@^7.3.0, espree@^7.3.1: + version "7.3.1" + resolved "https://registry.yarnpkg.com/espree/-/espree-7.3.1.tgz#f2df330b752c6f55019f8bd89b7660039c1bbbb6" + integrity sha512-v3JCNCE64umkFpmkFGqzVKsOT0tN1Zr+ueqLZfpV1Ob8e+CEgPWa+OxCoGH3tnhimMKIaBm4m/vaRpJ/krRz2g== + dependencies: + acorn "^7.4.0" + acorn-jsx "^5.3.1" + eslint-visitor-keys "^1.3.0" + +esprima@^4.0.0, esprima@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" + integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== + +esquery@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.4.0.tgz#2148ffc38b82e8c7057dfed48425b3e61f0f24a5" + integrity sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w== + dependencies: + estraverse "^5.1.0" + +esrecurse@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/esrecurse/-/esrecurse-4.3.0.tgz#7ad7964d679abb28bee72cec63758b1c5d2c9921" + integrity sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag== + dependencies: + estraverse "^5.2.0" + +estraverse@^4.1.1: + version "4.3.0" + resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-4.3.0.tgz#398ad3f3c5a24948be7725e83d11a7de28cdbd1d" + integrity sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw== + +estraverse@^5.1.0, estraverse@^5.2.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.2.0.tgz#307df42547e6cc7324d3cf03c155d5cdb8c53880" + integrity sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ== + +esutils@^2.0.2: + version "2.0.3" + resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" + integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== + +event-emitter@^0.3.5: + version "0.3.5" + resolved "https://registry.yarnpkg.com/event-emitter/-/event-emitter-0.3.5.tgz#df8c69eef1647923c7157b9ce83840610b02cc39" + integrity sha1-34xp7vFkeSPHFXuc6DhAYQsCzDk= + dependencies: + d "1" + es5-ext "~0.10.14" + +eventemitter3@^4.0.4: + version "4.0.7" + resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" + integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== + +execa@^5.0.0: + version "5.1.1" + resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd" + integrity sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg== + dependencies: + cross-spawn "^7.0.3" + get-stream "^6.0.0" + human-signals "^2.1.0" + is-stream "^2.0.0" + merge-stream "^2.0.0" + npm-run-path "^4.0.1" + onetime "^5.1.2" + signal-exit "^3.0.3" + strip-final-newline "^2.0.0" + +exit@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" + integrity sha1-BjJjj42HfMghB9MKD/8aF8uhzQw= + +expand-brackets@^2.1.4: + version "2.1.4" + resolved "https://registry.yarnpkg.com/expand-brackets/-/expand-brackets-2.1.4.tgz#b77735e315ce30f6b6eff0f83b04151a22449622" + integrity sha1-t3c14xXOMPa27/D4OwQVGiJEliI= + dependencies: + debug "^2.3.3" + define-property "^0.2.5" + extend-shallow "^2.0.1" + posix-character-classes "^0.1.0" + regex-not "^1.0.0" + snapdragon "^0.8.1" + to-regex "^3.0.1" + +expand-tilde@^2.0.0, expand-tilde@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/expand-tilde/-/expand-tilde-2.0.2.tgz#97e801aa052df02454de46b02bf621642cdc8502" + integrity sha1-l+gBqgUt8CRU3kawK/YhZCzchQI= + dependencies: + homedir-polyfill "^1.0.1" + +expect@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/expect/-/expect-27.0.6.tgz#a4d74fbe27222c718fff68ef49d78e26a8fd4c05" + integrity sha512-psNLt8j2kwg42jGBDSfAlU49CEZxejN1f1PlANWDZqIhBOVU/c2Pm888FcjWJzFewhIsNWfZJeLjUjtKGiPuSw== + dependencies: + "@jest/types" "^27.0.6" + ansi-styles "^5.0.0" + jest-get-type "^27.0.6" + jest-matcher-utils "^27.0.6" + jest-message-util "^27.0.6" + jest-regex-util "^27.0.6" + +ext@^1.1.2: + version "1.4.0" + resolved "https://registry.yarnpkg.com/ext/-/ext-1.4.0.tgz#89ae7a07158f79d35517882904324077e4379244" + integrity sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A== + dependencies: + type "^2.0.0" + +extend-shallow@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/extend-shallow/-/extend-shallow-2.0.1.tgz#51af7d614ad9a9f610ea1bafbb989d6b1c56890f" + integrity sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8= + dependencies: + is-extendable "^0.1.0" + +extend-shallow@^3.0.0, extend-shallow@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/extend-shallow/-/extend-shallow-3.0.2.tgz#26a71aaf073b39fb2127172746131c2704028db8" + integrity sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg= + dependencies: + assign-symbols "^1.0.0" + is-extendable "^1.0.1" + +extend@^3.0.0, extend@~3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" + integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== + +external-editor@^3.0.3: + version "3.1.0" + resolved "https://registry.yarnpkg.com/external-editor/-/external-editor-3.1.0.tgz#cb03f740befae03ea4d283caed2741a83f335495" + integrity sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew== + dependencies: + chardet "^0.7.0" + iconv-lite "^0.4.24" + tmp "^0.0.33" + +extglob@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/extglob/-/extglob-2.0.4.tgz#ad00fe4dc612a9232e8718711dc5cb5ab0285543" + integrity sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw== + dependencies: + array-unique "^0.3.2" + define-property "^1.0.0" + expand-brackets "^2.1.4" + extend-shallow "^2.0.1" + fragment-cache "^0.2.1" + regex-not "^1.0.0" + snapdragon "^0.8.1" + to-regex "^3.0.1" + +extsprintf@1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" + integrity sha1-lpGEQOMEGnpBT4xS48V06zw+HgU= + +extsprintf@^1.2.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f" + integrity sha1-4mifjzVvrWLMplo6kcXfX5VRaS8= + +fancy-log@^1.3.2: + version "1.3.3" + resolved "https://registry.yarnpkg.com/fancy-log/-/fancy-log-1.3.3.tgz#dbc19154f558690150a23953a0adbd035be45fc7" + integrity sha512-k9oEhlyc0FrVh25qYuSELjr8oxsCoc4/LEZfg2iJJrfEk/tZL9bCoJE47gqAvI2m/AUjluCS4+3I0eTx8n3AEw== + dependencies: + ansi-gray "^0.1.1" + color-support "^1.1.3" + parse-node-version "^1.0.0" + time-stamp "^1.0.0" + +fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: + version "3.1.3" + resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" + integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== + +fast-glob@^2.2.6: + version "2.2.7" + resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-2.2.7.tgz#6953857c3afa475fff92ee6015d52da70a4cd39d" + integrity sha512-g1KuQwHOZAmOZMuBtHdxDtju+T2RT8jgCC9aANsbpdiDDTSnjgfuVsIBNKbUeJI3oKMRExcfNDtJl4OhbffMsw== + dependencies: + "@mrmlnc/readdir-enhanced" "^2.2.1" + "@nodelib/fs.stat" "^1.1.2" + glob-parent "^3.1.0" + is-glob "^4.0.0" + merge2 "^1.2.3" + micromatch "^3.1.10" + +fast-glob@^3.1.1: + version "3.2.7" + resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.7.tgz#fd6cb7a2d7e9aa7a7846111e85a196d6b2f766a1" + integrity sha512-rYGMRwip6lUMvYD3BTScMwT1HtAs2d71SMv66Vrxs0IekGZEjhM0pcMfjQPnknBt2zeCwQMEupiN02ZP4DiT1Q== + dependencies: + "@nodelib/fs.stat" "^2.0.2" + "@nodelib/fs.walk" "^1.2.3" + glob-parent "^5.1.2" + merge2 "^1.3.0" + micromatch "^4.0.4" + +fast-json-stable-stringify@2.x, fast-json-stable-stringify@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" + integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== + +fast-levenshtein@^1.0.0: + version "1.1.4" + resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-1.1.4.tgz#e6a754cc8f15e58987aa9cbd27af66fd6f4e5af9" + integrity sha1-5qdUzI8V5YmHqpy9J69m/W9OWvk= + +fast-levenshtein@^2.0.6, fast-levenshtein@~2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" + integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc= + +fastq@^1.6.0: + version "1.11.1" + resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.11.1.tgz#5d8175aae17db61947f8b162cfc7f63264d22807" + integrity sha512-HOnr8Mc60eNYl1gzwp6r5RoUyAn5/glBolUzP/Ez6IFVPMPirxn/9phgL6zhOtaTy7ISwPvQ+wT+hfcRZh/bzw== + dependencies: + reusify "^1.0.4" + +fb-watchman@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/fb-watchman/-/fb-watchman-2.0.1.tgz#fc84fb39d2709cf3ff6d743706157bb5708a8a85" + integrity sha512-DkPJKQeY6kKwmuMretBhr7G6Vodr7bFwDYTXIkfG1gjvNpaxBTQV3PbXg6bR1c1UP4jPOX0jHUbbHANL9vRjVg== + dependencies: + bser "2.1.1" + +figures@^3.0.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" + integrity sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg== + dependencies: + escape-string-regexp "^1.0.5" + +file-entry-cache@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-6.0.1.tgz#211b2dd9659cb0394b073e7323ac3c933d522027" + integrity sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg== + dependencies: + flat-cache "^3.0.4" + +file-uri-to-path@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" + integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== + +fill-range@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-4.0.0.tgz#d544811d428f98eb06a63dc402d2403c328c38f7" + integrity sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc= + dependencies: + extend-shallow "^2.0.1" + is-number "^3.0.0" + repeat-string "^1.6.1" + to-regex-range "^2.1.0" + +fill-range@^7.0.1: + version "7.0.1" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40" + integrity sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ== + dependencies: + to-regex-range "^5.0.1" + +filter-obj@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/filter-obj/-/filter-obj-1.1.0.tgz#9b311112bc6c6127a16e016c6c5d7f19e0805c5b" + integrity sha1-mzERErxsYSehbgFsbF1/GeCAXFs= + +find-replace@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/find-replace/-/find-replace-3.0.0.tgz#3e7e23d3b05167a76f770c9fbd5258b0def68c38" + integrity sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ== + dependencies: + array-back "^3.0.1" + +find-up@^1.0.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-1.1.2.tgz#6b2e9822b1a2ce0a60ab64d610eccad53cb24d0f" + integrity sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8= + dependencies: + path-exists "^2.0.0" + pinkie-promise "^2.0.0" + +find-up@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-2.1.0.tgz#45d1b7e506c717ddd482775a2b77920a3c0c57a7" + integrity sha1-RdG35QbHF93UgndaK3eSCjwMV6c= + dependencies: + locate-path "^2.0.0" + +find-up@^4.0.0, find-up@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-4.1.0.tgz#97afe7d6cdc0bc5928584b7c8d7b16e8a9aa5d19" + integrity sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw== + dependencies: + locate-path "^5.0.0" + path-exists "^4.0.0" + +find-up@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" + integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== + dependencies: + locate-path "^6.0.0" + path-exists "^4.0.0" + +findup-sync@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-2.0.0.tgz#9326b1488c22d1a6088650a86901b2d9a90a2cbc" + integrity sha1-kyaxSIwi0aYIhlCoaQGy2akKLLw= + dependencies: + detect-file "^1.0.0" + is-glob "^3.1.0" + micromatch "^3.0.4" + resolve-dir "^1.0.1" + +findup-sync@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-3.0.0.tgz#17b108f9ee512dfb7a5c7f3c8b27ea9e1a9c08d1" + integrity sha512-YbffarhcicEhOrm4CtrwdKBdCuz576RLdhJDsIfvNtxUuhdRet1qZcsMjqbePtAseKdAnDyM/IyXbu7PRPRLYg== + dependencies: + detect-file "^1.0.0" + is-glob "^4.0.0" + micromatch "^3.0.4" + resolve-dir "^1.0.1" + +fined@^1.0.1: + version "1.2.0" + resolved "https://registry.yarnpkg.com/fined/-/fined-1.2.0.tgz#d00beccf1aa2b475d16d423b0238b713a2c4a37b" + integrity sha512-ZYDqPLGxDkDhDZBjZBb+oD1+j0rA4E0pXY50eplAAOPg2N/gUBSSk5IM1/QhPfyVo19lJ+CvXpqfvk+b2p/8Ng== + dependencies: + expand-tilde "^2.0.2" + is-plain-object "^2.0.3" + object.defaults "^1.1.0" + object.pick "^1.2.0" + parse-filepath "^1.0.1" + +flagged-respawn@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/flagged-respawn/-/flagged-respawn-1.0.1.tgz#e7de6f1279ddd9ca9aac8a5971d618606b3aab41" + integrity sha512-lNaHNVymajmk0OJMBn8fVUAU1BtDeKIqKoVhk4xAALB57aALg6b4W0MfJ/cUE0g9YBXy5XhSlPIpYIJ7HaY/3Q== + +flat-cache@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-3.0.4.tgz#61b0338302b2fe9f957dcc32fc2a87f1c3048b11" + integrity sha512-dm9s5Pw7Jc0GvMYbshN6zchCA9RgQlzzEZX3vylR9IqFfS8XciblUXOKfW6SiuJ0e13eDYZoZV5wdrev7P3Nwg== + dependencies: + flatted "^3.1.0" + rimraf "^3.0.2" + +flatbuffers@1.12.0: + version "1.12.0" + resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-1.12.0.tgz#72e87d1726cb1b216e839ef02658aa87dcef68aa" + integrity sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ== + +flatted@^3.1.0: + version "3.2.1" + resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.1.tgz#bbef080d95fca6709362c73044a1634f7c6e7d05" + integrity sha512-OMQjaErSFHmHqZe+PSidH5n8j3O0F2DdnVh8JB4j4eUQ2k6KvB0qGfrKIhapvez5JerBbmWkaLYUYWISaESoXg== + +flush-write-stream@^1.0.2: + version "1.1.1" + resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8" + integrity sha512-3Z4XhFZ3992uIq0XOqb9AreonueSYphE6oYbpt5+3u06JWklbsPkNv3ZKkP9Bz/r+1MWCaMoSQ28P85+1Yc77w== + dependencies: + inherits "^2.0.3" + readable-stream "^2.3.6" + +for-in@^1.0.1, for-in@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" + integrity sha1-gQaNKVqBQuwKxybG4iAMMPttXoA= + +for-own@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/for-own/-/for-own-1.0.0.tgz#c63332f415cedc4b04dbfe70cf836494c53cb44b" + integrity sha1-xjMy9BXO3EsE2/5wz4NklMU8tEs= + dependencies: + for-in "^1.0.1" + +forever-agent@~0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" + integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE= + +form-data@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.1.tgz#ebd53791b78356a99af9a300d4282c4d5eb9755f" + integrity sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + +form-data@~2.3.2: + version "2.3.3" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" + integrity sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.6" + mime-types "^2.1.12" + +fragment-cache@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/fragment-cache/-/fragment-cache-0.2.1.tgz#4290fad27f13e89be7f33799c6bc5a0abfff0d19" + integrity sha1-QpD60n8T6Jvn8zeZxrxaCr//DRk= + dependencies: + map-cache "^0.2.2" + +fs-extra@^9.0.1, fs-extra@^9.1.0: + version "9.1.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" + integrity sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ== + dependencies: + at-least-node "^1.0.0" + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" + +fs-minipass@^1.2.5: + version "1.2.7" + resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-1.2.7.tgz#ccff8570841e7fe4265693da88936c55aed7f7c7" + integrity sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA== + dependencies: + minipass "^2.6.0" + +fs-minipass@^2.0.0, fs-minipass@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-2.1.0.tgz#7f5036fdbf12c63c169190cbe4199c852271f9fb" + integrity sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg== + dependencies: + minipass "^3.0.0" + +fs-mkdirp-stream@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs-mkdirp-stream/-/fs-mkdirp-stream-1.0.0.tgz#0b7815fc3201c6a69e14db98ce098c16935259eb" + integrity sha1-C3gV/DIBxqaeFNuYzgmMFpNSWes= + dependencies: + graceful-fs "^4.1.11" + through2 "^2.0.3" + +fs-monkey@1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/fs-monkey/-/fs-monkey-1.0.3.tgz#ae3ac92d53bb328efe0e9a1d9541f6ad8d48e2d3" + integrity sha512-cybjIfiiE+pTWicSCLFHSrXZ6EilF30oh91FDP9S2B051prEa7QWfrVTQm10/dDpswBDXZugPa1Ogu8Yh+HV0Q== + +fs.realpath@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" + integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8= + +fsevents@^1.2.7: + version "1.2.13" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-1.2.13.tgz#f325cb0455592428bcf11b383370ef70e3bfcc38" + integrity sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw== + dependencies: + bindings "^1.5.0" + nan "^2.12.1" + +fsevents@^2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" + integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== + +function-bind@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" + integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== + +functional-red-black-tree@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327" + integrity sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc= + +gauge@~2.7.3: + version "2.7.4" + resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7" + integrity sha1-LANAXHU4w51+s3sxcCLjJfsBi/c= + dependencies: + aproba "^1.0.3" + console-control-strings "^1.0.0" + has-unicode "^2.0.0" + object-assign "^4.1.0" + signal-exit "^3.0.0" + string-width "^1.0.1" + strip-ansi "^3.0.1" + wide-align "^1.1.0" + +gensync@^1.0.0-beta.2: + version "1.0.0-beta.2" + resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" + integrity sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg== + +get-caller-file@^1.0.1: + version "1.0.3" + resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.3.tgz#f978fa4c90d1dfe7ff2d6beda2a515e713bdcf4a" + integrity sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w== + +get-caller-file@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" + integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== + +get-intrinsic@^1.0.2, get-intrinsic@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.1.tgz#15f59f376f855c446963948f0d24cd3637b4abc6" + integrity sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q== + dependencies: + function-bind "^1.1.1" + has "^1.0.3" + has-symbols "^1.0.1" + +get-package-type@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" + integrity sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q== + +get-pkg-repo@^4.0.0: + version "4.1.2" + resolved "https://registry.yarnpkg.com/get-pkg-repo/-/get-pkg-repo-4.1.2.tgz#c4ffd60015cf091be666a0212753fc158f01a4c0" + integrity sha512-/FjamZL9cBYllEbReZkxF2IMh80d8TJoC4e3bmLNif8ibHw95aj0N/tzqK0kZz9eU/3w3dL6lF4fnnX/sDdW3A== + dependencies: + "@hutson/parse-repository-url" "^3.0.0" + hosted-git-info "^4.0.0" + meow "^7.0.0" + through2 "^2.0.0" + +get-port@^5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/get-port/-/get-port-5.1.1.tgz#0469ed07563479de6efb986baf053dcd7d4e3193" + integrity sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ== + +get-stream@^6.0.0: + version "6.0.1" + resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.1.tgz#a262d8eef67aced57c2852ad6167526a43cbf7b7" + integrity sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg== + +get-value@^2.0.3, get-value@^2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28" + integrity sha1-3BXKHGcjh8p2vTesCjlbogQqLCg= + +getpass@^0.1.1: + version "0.1.7" + resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" + integrity sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo= + dependencies: + assert-plus "^1.0.0" + +git-raw-commits@^2.0.8: + version "2.0.10" + resolved "https://registry.yarnpkg.com/git-raw-commits/-/git-raw-commits-2.0.10.tgz#e2255ed9563b1c9c3ea6bd05806410290297bbc1" + integrity sha512-sHhX5lsbG9SOO6yXdlwgEMQ/ljIn7qMpAbJZCGfXX2fq5T8M5SrDnpYk9/4HswTildcIqatsWa91vty6VhWSaQ== + dependencies: + dargs "^7.0.0" + lodash "^4.17.15" + meow "^8.0.0" + split2 "^3.0.0" + through2 "^4.0.0" + +git-remote-origin-url@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/git-remote-origin-url/-/git-remote-origin-url-2.0.0.tgz#5282659dae2107145a11126112ad3216ec5fa65f" + integrity sha1-UoJlna4hBxRaERJhEq0yFuxfpl8= + dependencies: + gitconfiglocal "^1.0.0" + pify "^2.3.0" + +git-semver-tags@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/git-semver-tags/-/git-semver-tags-4.1.1.tgz#63191bcd809b0ec3e151ba4751c16c444e5b5780" + integrity sha512-OWyMt5zBe7xFs8vglMmhM9lRQzCWL3WjHtxNNfJTMngGym7pC1kh8sP6jevfydJ6LP3ZvGxfb6ABYgPUM0mtsA== + dependencies: + meow "^8.0.0" + semver "^6.0.0" + +git-up@^4.0.0: + version "4.0.5" + resolved "https://registry.yarnpkg.com/git-up/-/git-up-4.0.5.tgz#e7bb70981a37ea2fb8fe049669800a1f9a01d759" + integrity sha512-YUvVDg/vX3d0syBsk/CKUTib0srcQME0JyHkL5BaYdwLsiCslPWmDSi8PUMo9pXYjrryMcmsCoCgsTpSCJEQaA== + dependencies: + is-ssh "^1.3.0" + parse-url "^6.0.0" + +git-url-parse@^11.4.4: + version "11.5.0" + resolved "https://registry.yarnpkg.com/git-url-parse/-/git-url-parse-11.5.0.tgz#acaaf65239cb1536185b19165a24bbc754b3f764" + integrity sha512-TZYSMDeM37r71Lqg1mbnMlOqlHd7BSij9qN7XwTkRqSAYFMihGLGhfHwgqQob3GUhEneKnV4nskN9rbQw2KGxA== + dependencies: + git-up "^4.0.0" + +gitconfiglocal@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/gitconfiglocal/-/gitconfiglocal-1.0.0.tgz#41d045f3851a5ea88f03f24ca1c6178114464b9b" + integrity sha1-QdBF84UaXqiPA/JMocYXgRRGS5s= + dependencies: + ini "^1.3.2" + +glob-parent@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-3.1.0.tgz#9e6af6299d8d3bd2bd40430832bd113df906c5ae" + integrity sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4= + dependencies: + is-glob "^3.1.0" + path-dirname "^1.0.0" + +glob-parent@^5.1.1, glob-parent@^5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" + integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== + dependencies: + is-glob "^4.0.1" + +glob-stream@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/glob-stream/-/glob-stream-6.1.0.tgz#7045c99413b3eb94888d83ab46d0b404cc7bdde4" + integrity sha1-cEXJlBOz65SIjYOrRtC0BMx73eQ= + dependencies: + extend "^3.0.0" + glob "^7.1.1" + glob-parent "^3.1.0" + is-negated-glob "^1.0.0" + ordered-read-streams "^1.0.0" + pumpify "^1.3.5" + readable-stream "^2.1.5" + remove-trailing-separator "^1.0.1" + to-absolute-glob "^2.0.0" + unique-stream "^2.0.2" + +glob-to-regexp@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab" + integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs= + +glob-watcher@^5.0.3: + version "5.0.5" + resolved "https://registry.yarnpkg.com/glob-watcher/-/glob-watcher-5.0.5.tgz#aa6bce648332924d9a8489be41e3e5c52d4186dc" + integrity sha512-zOZgGGEHPklZNjZQaZ9f41i7F2YwE+tS5ZHrDhbBCk3stwahn5vQxnFmBJZHoYdusR6R1bLSXeGUy/BhctwKzw== + dependencies: + anymatch "^2.0.0" + async-done "^1.2.0" + chokidar "^2.0.0" + is-negated-glob "^1.0.0" + just-debounce "^1.0.0" + normalize-path "^3.0.0" + object.defaults "^1.1.0" + +glob@7.1.7, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@^7.1.7: + version "7.1.7" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.7.tgz#3b193e9233f01d42d0b3f78294bbeeb418f94a90" + integrity sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ== + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + +global-modules@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-1.0.0.tgz#6d770f0eb523ac78164d72b5e71a8877265cc3ea" + integrity sha512-sKzpEkf11GpOFuw0Zzjzmt4B4UZwjOcG757PPvrfhxcLFbq0wpsgpOqxpxtxFiCG4DtG93M6XRVbF2oGdev7bg== + dependencies: + global-prefix "^1.0.1" + is-windows "^1.0.1" + resolve-dir "^1.0.0" + +global-prefix@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/global-prefix/-/global-prefix-1.0.2.tgz#dbf743c6c14992593c655568cb66ed32c0122ebe" + integrity sha1-2/dDxsFJklk8ZVVoy2btMsASLr4= + dependencies: + expand-tilde "^2.0.2" + homedir-polyfill "^1.0.1" + ini "^1.3.4" + is-windows "^1.0.1" + which "^1.2.14" + +globals@^11.1.0: + version "11.12.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-11.12.0.tgz#ab8795338868a0babd8525758018c2a7eb95c42e" + integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== + +globals@^13.6.0, globals@^13.9.0: + version "13.10.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-13.10.0.tgz#60ba56c3ac2ca845cfbf4faeca727ad9dd204676" + integrity sha512-piHC3blgLGFjvOuMmWZX60f+na1lXFDhQXBf1UYp2fXPXqvEUbOhNwi6BsQ0bQishwedgnjkwv1d9zKf+MWw3g== + dependencies: + type-fest "^0.20.2" + +globby@^11.0.1, globby@^11.0.2, globby@^11.0.3: + version "11.0.4" + resolved "https://registry.yarnpkg.com/globby/-/globby-11.0.4.tgz#2cbaff77c2f2a62e71e9b2813a67b97a3a3001a5" + integrity sha512-9O4MVG9ioZJ08ffbcyVYyLOJLk5JQ688pJ4eMGLpdWLHq/Wr1D9BlriLQyL0E+jbkuePVZXYFj47QM/v093wHg== + dependencies: + array-union "^2.1.0" + dir-glob "^3.0.1" + fast-glob "^3.1.1" + ignore "^5.1.4" + merge2 "^1.3.0" + slash "^3.0.0" + +globby@^9.2.0: + version "9.2.0" + resolved "https://registry.yarnpkg.com/globby/-/globby-9.2.0.tgz#fd029a706c703d29bdd170f4b6db3a3f7a7cb63d" + integrity sha512-ollPHROa5mcxDEkwg6bPt3QbEf4pDQSNtd6JPL1YvOvAo/7/0VAm9TccUeoTmarjPw4pfUthSCqcyfNB1I3ZSg== + dependencies: + "@types/glob" "^7.1.1" + array-union "^1.0.2" + dir-glob "^2.2.2" + fast-glob "^2.2.6" + glob "^7.1.3" + ignore "^4.0.3" + pify "^4.0.1" + slash "^2.0.0" + +glogg@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/glogg/-/glogg-1.0.2.tgz#2d7dd702beda22eb3bffadf880696da6d846313f" + integrity sha512-5mwUoSuBk44Y4EshyiqcH95ZntbDdTQqA3QYSrxmzj28Ai0vXBGMH1ApSANH14j2sIRtqCEyg6PfsuP7ElOEDA== + dependencies: + sparkles "^1.0.0" + +google-closure-compiler-java@^20210601.0.0: + version "20210601.0.0" + resolved "https://registry.yarnpkg.com/google-closure-compiler-java/-/google-closure-compiler-java-20210601.0.0.tgz#88dc11b334bee6a704d9674c5143fd2e0d553517" + integrity sha512-bH6nIwOmp4qDWvlbXx5/DE3XA2aDGQoCpmRYZJGONY1Sy6Xfbq0ioXRHH9eBDP9hxhCJ5Sd/K89A0NZ8Nz9RJA== + +google-closure-compiler-linux@^20210601.0.0: + version "20210601.0.0" + resolved "https://registry.yarnpkg.com/google-closure-compiler-linux/-/google-closure-compiler-linux-20210601.0.0.tgz#6e5dd7b00b96dc1fd1ba30e3401af85558768322" + integrity sha512-rnEQt7zz/1P1SfPhJiHQpfCgMPrsVVyEgDs09h67xn6+LXa9L0RP+hrJDEHqSWwjDPz0BkfUUv6zkqZvp1h/lw== + +google-closure-compiler-osx@^20210601.0.0: + version "20210601.0.0" + resolved "https://registry.yarnpkg.com/google-closure-compiler-osx/-/google-closure-compiler-osx-20210601.0.0.tgz#e23356bc9ef6e68c2980f60a207f603767b50b21" + integrity sha512-A5r4s/WthR2iLMM0mxsluw8EW2AcOomC5ri/H6FjzpMq0RVEnLTgaGYdXolUAfEzH/7XtJJT2+JkYk3HSLCtrg== + +google-closure-compiler-windows@^20210601.0.0: + version "20210601.0.0" + resolved "https://registry.yarnpkg.com/google-closure-compiler-windows/-/google-closure-compiler-windows-20210601.0.0.tgz#b5400d06bbf0bbd2602ee3ae0c2bc7ebd5829692" + integrity sha512-6r94bPShnB0XXh9+5/qXGDHJN2PQGhF9yJPcgBZj+FAZlQGzlYkT0pkyp+loZT3lG+YRbjD28Lgo7xMcY4xgkA== + +google-closure-compiler@20210601.0.0: + version "20210601.0.0" + resolved "https://registry.yarnpkg.com/google-closure-compiler/-/google-closure-compiler-20210601.0.0.tgz#34597c33c9285ebd3a5364f5299f6c9ddc9fc88a" + integrity sha512-lzzEoG2VTB7uUjnWnMyeZMU163w69HJpM27yh8Up9Ha5McHZeESjt3NRwU8cWMbCRdY06nFbRCDIVCRcadHCiw== + dependencies: + chalk "2.x" + google-closure-compiler-java "^20210601.0.0" + minimist "1.x" + vinyl "2.x" + vinyl-sourcemaps-apply "^0.2.0" + optionalDependencies: + google-closure-compiler-linux "^20210601.0.0" + google-closure-compiler-osx "^20210601.0.0" + google-closure-compiler-windows "^20210601.0.0" + +graceful-fs@^4.0.0, graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.2, graceful-fs@^4.2.3, graceful-fs@^4.2.4: + version "4.2.6" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.6.tgz#ff040b2b0853b23c3d31027523706f1885d76bee" + integrity sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ== + +gulp-cli@^2.2.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/gulp-cli/-/gulp-cli-2.3.0.tgz#ec0d380e29e52aa45e47977f0d32e18fd161122f" + integrity sha512-zzGBl5fHo0EKSXsHzjspp3y5CONegCm8ErO5Qh0UzFzk2y4tMvzLWhoDokADbarfZRL2pGpRp7yt6gfJX4ph7A== + dependencies: + ansi-colors "^1.0.1" + archy "^1.0.0" + array-sort "^1.0.0" + color-support "^1.1.3" + concat-stream "^1.6.0" + copy-props "^2.0.1" + fancy-log "^1.3.2" + gulplog "^1.0.0" + interpret "^1.4.0" + isobject "^3.0.1" + liftoff "^3.1.0" + matchdep "^2.0.0" + mute-stdout "^1.0.0" + pretty-hrtime "^1.0.0" + replace-homedir "^1.0.0" + semver-greatest-satisfied-range "^1.1.0" + v8flags "^3.2.0" + yargs "^7.1.0" + +gulp-json-transform@0.4.7: + version "0.4.7" + resolved "https://registry.yarnpkg.com/gulp-json-transform/-/gulp-json-transform-0.4.7.tgz#41c37524c976e41f3d46c06f985b01530a472e34" + integrity sha512-Wi0p5GpoLXbTDwaZnw6rgj3FMLW3PscaHaX1okxrTgPWeqnIiMo4aJz7VlG68JYkxPeAXJrPce8AGEfcT2IifA== + dependencies: + ansi-colors "^1.0.1" + fancy-log "^1.3.2" + plugin-error "^1.0.1" + promise "^8.0.1" + through2 "^2.0.3" + vinyl "^2.1.0" + +gulp-rename@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/gulp-rename/-/gulp-rename-2.0.0.tgz#9bbc3962b0c0f52fc67cd5eaff6c223ec5b9cf6c" + integrity sha512-97Vba4KBzbYmR5VBs9mWmK+HwIf5mj+/zioxfZhOKeXtx5ZjBk57KFlePf5nxq9QsTtFl0ejnHE3zTC9MHXqyQ== + +gulp-sourcemaps@3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/gulp-sourcemaps/-/gulp-sourcemaps-3.0.0.tgz#2e154e1a2efed033c0e48013969e6f30337b2743" + integrity sha512-RqvUckJkuYqy4VaIH60RMal4ZtG0IbQ6PXMNkNsshEGJ9cldUPRb/YCgboYae+CLAs1HQNb4ADTKCx65HInquQ== + dependencies: + "@gulp-sourcemaps/identity-map" "^2.0.1" + "@gulp-sourcemaps/map-sources" "^1.0.0" + acorn "^6.4.1" + convert-source-map "^1.0.0" + css "^3.0.0" + debug-fabulous "^1.0.0" + detect-newline "^2.0.0" + graceful-fs "^4.0.0" + source-map "^0.6.0" + strip-bom-string "^1.0.0" + through2 "^2.0.0" + +gulp-typescript@5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/gulp-typescript/-/gulp-typescript-5.0.1.tgz#96c6565a6eb31e08c2aae1c857b1a079e6226d94" + integrity sha512-YuMMlylyJtUSHG1/wuSVTrZp60k1dMEFKYOvDf7OvbAJWrDtxxD4oZon4ancdWwzjj30ztiidhe4VXJniF0pIQ== + dependencies: + ansi-colors "^3.0.5" + plugin-error "^1.0.1" + source-map "^0.7.3" + through2 "^3.0.0" + vinyl "^2.1.0" + vinyl-fs "^3.0.3" + +gulp@4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/gulp/-/gulp-4.0.2.tgz#543651070fd0f6ab0a0650c6a3e6ff5a7cb09caa" + integrity sha512-dvEs27SCZt2ibF29xYgmnwwCYZxdxhQ/+LFWlbAW8y7jt68L/65402Lz3+CKy0Ov4rOs+NERmDq7YlZaDqUIfA== + dependencies: + glob-watcher "^5.0.3" + gulp-cli "^2.2.0" + undertaker "^1.2.1" + vinyl-fs "^3.0.0" + +gulplog@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/gulplog/-/gulplog-1.0.0.tgz#e28c4d45d05ecbbed818363ce8f9c5926229ffe5" + integrity sha1-4oxNRdBey77YGDY86PnFkmIp/+U= + dependencies: + glogg "^1.0.0" + +handlebars@^4.7.6, handlebars@^4.7.7: + version "4.7.7" + resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.7.tgz#9ce33416aad02dbd6c8fafa8240d5d98004945a1" + integrity sha512-aAcXm5OAfE/8IXkcZvCepKU3VzW1/39Fb5ZuqMtgI/hT8X2YgoMvBY5dLhq/cpOvw7Lk1nK/UF71aLG/ZnVYRA== + dependencies: + minimist "^1.2.5" + neo-async "^2.6.0" + source-map "^0.6.1" + wordwrap "^1.0.0" + optionalDependencies: + uglify-js "^3.1.4" + +har-schema@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" + integrity sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI= + +har-validator@~5.1.3: + version "5.1.5" + resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.1.5.tgz#1f0803b9f8cb20c0fa13822df1ecddb36bde1efd" + integrity sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w== + dependencies: + ajv "^6.12.3" + har-schema "^2.0.0" + +hard-rejection@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/hard-rejection/-/hard-rejection-2.1.0.tgz#1c6eda5c1685c63942766d79bb40ae773cecd883" + integrity sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA== + +has-bigints@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/has-bigints/-/has-bigints-1.0.1.tgz#64fe6acb020673e3b78db035a5af69aa9d07b113" + integrity sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA== + +has-flag@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" + integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0= + +has-flag@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" + integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== + +has-glob@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/has-glob/-/has-glob-1.0.0.tgz#9aaa9eedbffb1ba3990a7b0010fb678ee0081207" + integrity sha1-mqqe7b/7G6OZCnsAEPtnjuAIEgc= + dependencies: + is-glob "^3.0.0" + +has-symbols@^1.0.1, has-symbols@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.2.tgz#165d3070c00309752a1236a479331e3ac56f1423" + integrity sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw== + +has-unicode@^2.0.0, has-unicode@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" + integrity sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk= + +has-value@^0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/has-value/-/has-value-0.3.1.tgz#7b1f58bada62ca827ec0a2078025654845995e1f" + integrity sha1-ex9YutpiyoJ+wKIHgCVlSEWZXh8= + dependencies: + get-value "^2.0.3" + has-values "^0.1.4" + isobject "^2.0.0" + +has-value@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/has-value/-/has-value-1.0.0.tgz#18b281da585b1c5c51def24c930ed29a0be6b177" + integrity sha1-GLKB2lhbHFxR3vJMkw7SmgvmsXc= + dependencies: + get-value "^2.0.6" + has-values "^1.0.0" + isobject "^3.0.0" + +has-values@^0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/has-values/-/has-values-0.1.4.tgz#6d61de95d91dfca9b9a02089ad384bff8f62b771" + integrity sha1-bWHeldkd/Km5oCCJrThL/49it3E= + +has-values@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/has-values/-/has-values-1.0.0.tgz#95b0b63fec2146619a6fe57fe75628d5a39efe4f" + integrity sha1-lbC2P+whRmGab+V/51Yo1aOe/k8= + dependencies: + is-number "^3.0.0" + kind-of "^4.0.0" + +has@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" + integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw== + dependencies: + function-bind "^1.1.1" + +homedir-polyfill@^1.0.1: + version "1.0.3" + resolved "https://registry.yarnpkg.com/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz#743298cef4e5af3e194161fbadcc2151d3a058e8" + integrity sha512-eSmmWE5bZTK2Nou4g0AI3zZ9rswp7GRKoKXS1BLUkvPviOqs4YTN1djQIqrXy9k5gEtdLPy86JjRwsNM9tnDcA== + dependencies: + parse-passwd "^1.0.0" + +hosted-git-info@^2.1.4: + version "2.8.9" + resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9" + integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw== + +hosted-git-info@^4.0.0, hosted-git-info@^4.0.1: + version "4.0.2" + resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-4.0.2.tgz#5e425507eede4fea846b7262f0838456c4209961" + integrity sha512-c9OGXbZ3guC/xOlCg1Ci/VgWlwsqDv1yMQL1CWqXDL0hDjXuNcq0zuR4xqPSuasI3kqFDhqSyTjREz5gzq0fXg== + dependencies: + lru-cache "^6.0.0" + +html-encoding-sniffer@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/html-encoding-sniffer/-/html-encoding-sniffer-2.0.1.tgz#42a6dc4fd33f00281176e8b23759ca4e4fa185f3" + integrity sha512-D5JbOMBIR/TVZkubHT+OyT2705QvogUW4IBn6nHd756OwieSF9aDYFj4dv6HHEVGYbHaLETa3WggZYWWMyy3ZQ== + dependencies: + whatwg-encoding "^1.0.5" + +html-escaper@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" + integrity sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg== + +http-cache-semantics@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz#49e91c5cbf36c9b94bcfcd71c23d5249ec74e390" + integrity sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ== + +http-proxy-agent@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz#8a8c8ef7f5932ccf953c296ca8291b95aa74aa3a" + integrity sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg== + dependencies: + "@tootallnate/once" "1" + agent-base "6" + debug "4" + +http-signature@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" + integrity sha1-muzZJRFHcvPZW2WmCruPfBj7rOE= + dependencies: + assert-plus "^1.0.0" + jsprim "^1.2.2" + sshpk "^1.7.0" + +https-proxy-agent@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz#e2a90542abb68a762e0a0850f6c9edadfd8506b2" + integrity sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA== + dependencies: + agent-base "6" + debug "4" + +human-signals@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" + integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== + +humanize-ms@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/humanize-ms/-/humanize-ms-1.2.1.tgz#c46e3159a293f6b896da29316d8b6fe8bb79bbed" + integrity sha1-xG4xWaKT9riW2ikxbYtv6Lt5u+0= + dependencies: + ms "^2.0.0" + +iconv-lite@0.4.24, iconv-lite@^0.4.24: + version "0.4.24" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" + integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== + dependencies: + safer-buffer ">= 2.1.2 < 3" + +iconv-lite@^0.6.2: + version "0.6.3" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501" + integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== + dependencies: + safer-buffer ">= 2.1.2 < 3.0.0" + +ignore-walk@^3.0.3: + version "3.0.4" + resolved "https://registry.yarnpkg.com/ignore-walk/-/ignore-walk-3.0.4.tgz#c9a09f69b7c7b479a5d74ac1a3c0d4236d2a6335" + integrity sha512-PY6Ii8o1jMRA1z4F2hRkH/xN59ox43DavKvD3oDpfurRlOJyAHpifIwpbdv1n4jt4ov0jSpw3kQ4GhJnpBL6WQ== + dependencies: + minimatch "^3.0.4" + +ignore@^4.0.3, ignore@^4.0.6: + version "4.0.6" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-4.0.6.tgz#750e3db5862087b4737ebac8207ffd1ef27b25fc" + integrity sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg== + +ignore@^5.1.4: + version "5.1.8" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.1.8.tgz#f150a8b50a34289b33e22f5889abd4d8016f0e57" + integrity sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw== + +import-fresh@^3.0.0, import-fresh@^3.2.1: + version "3.3.0" + resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b" + integrity sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw== + dependencies: + parent-module "^1.0.0" + resolve-from "^4.0.0" + +import-local@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/import-local/-/import-local-3.0.2.tgz#a8cfd0431d1de4a2199703d003e3e62364fa6db6" + integrity sha512-vjL3+w0oulAVZ0hBHnxa/Nm5TAurf9YLQJDhqRZyqb+VKGOB6LU8t9H1Nr5CIo16vh9XfJTOoHwU0B71S557gA== + dependencies: + pkg-dir "^4.2.0" + resolve-cwd "^3.0.0" + +imurmurhash@^0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" + integrity sha1-khi5srkoojixPcT7a21XbyMUU+o= + +indent-string@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-4.0.0.tgz#624f8f4497d619b2d9768531d58f4122854d7251" + integrity sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg== + +indent-string@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-5.0.0.tgz#4fd2980fccaf8622d14c64d694f4cf33c81951a5" + integrity sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg== + +infer-owner@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/infer-owner/-/infer-owner-1.0.4.tgz#c4cefcaa8e51051c2a40ba2ce8a3d27295af9467" + integrity sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A== + +inflight@^1.0.4: + version "1.0.6" + resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" + integrity sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk= + dependencies: + once "^1.3.0" + wrappy "1" + +inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3: + version "2.0.4" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" + integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== + +ini@^1.3.2, ini@^1.3.4: + version "1.3.8" + resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" + integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== + +init-package-json@^2.0.2: + version "2.0.3" + resolved "https://registry.yarnpkg.com/init-package-json/-/init-package-json-2.0.3.tgz#c8ae4f2a4ad353bcbc089e5ffe98a8f1a314e8fd" + integrity sha512-tk/gAgbMMxR6fn1MgMaM1HpU1ryAmBWWitnxG5OhuNXeX0cbpbgV5jA4AIpQJVNoyOfOevTtO6WX+rPs+EFqaQ== + dependencies: + glob "^7.1.1" + npm-package-arg "^8.1.2" + promzard "^0.3.0" + read "~1.0.1" + read-package-json "^3.0.1" + semver "^7.3.5" + validate-npm-package-license "^3.0.4" + validate-npm-package-name "^3.0.0" + +inquirer@^7.3.3: + version "7.3.3" + resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-7.3.3.tgz#04d176b2af04afc157a83fd7c100e98ee0aad003" + integrity sha512-JG3eIAj5V9CwcGvuOmoo6LB9kbAYT8HXffUl6memuszlwDC/qvFAJw49XJ5NROSFNPxp3iQg1GqkFhaY/CR0IA== + dependencies: + ansi-escapes "^4.2.1" + chalk "^4.1.0" + cli-cursor "^3.1.0" + cli-width "^3.0.0" + external-editor "^3.0.3" + figures "^3.0.0" + lodash "^4.17.19" + mute-stream "0.0.8" + run-async "^2.4.0" + rxjs "^6.6.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + through "^2.3.6" + +interpret@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.4.0.tgz#665ab8bc4da27a774a40584e812e3e0fa45b1a1e" + integrity sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA== + +invert-kv@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-1.0.0.tgz#104a8e4aaca6d3d8cd157a8ef8bfab2d7a3ffdb6" + integrity sha1-EEqOSqym09jNFXqO+L+rLXo//bY= + +ip@^1.1.5: + version "1.1.5" + resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.5.tgz#bdded70114290828c0a039e72ef25f5aaec4354a" + integrity sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo= + +is-absolute@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-absolute/-/is-absolute-1.0.0.tgz#395e1ae84b11f26ad1795e73c17378e48a301576" + integrity sha512-dOWoqflvcydARa360Gvv18DZ/gRuHKi2NU/wU5X1ZFzdYfH29nkiNZsF3mp4OJ3H4yo9Mx8A/uAGNzpzPN3yBA== + dependencies: + is-relative "^1.0.0" + is-windows "^1.0.1" + +is-accessor-descriptor@^0.1.6: + version "0.1.6" + resolved "https://registry.yarnpkg.com/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz#a9e12cb3ae8d876727eeef3843f8a0897b5c98d6" + integrity sha1-qeEss66Nh2cn7u84Q/igiXtcmNY= + dependencies: + kind-of "^3.0.2" + +is-accessor-descriptor@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz#169c2f6d3df1f992618072365c9b0ea1f6878656" + integrity sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ== + dependencies: + kind-of "^6.0.0" + +is-arrayish@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" + integrity sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0= + +is-bigint@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-bigint/-/is-bigint-1.0.2.tgz#ffb381442503235ad245ea89e45b3dbff040ee5a" + integrity sha512-0JV5+SOCQkIdzjBK9buARcV804Ddu7A0Qet6sHi3FimE9ne6m4BGQZfRn+NZiXbBk4F4XmHfDZIipLj9pX8dSA== + +is-binary-path@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-1.0.1.tgz#75f16642b480f187a711c814161fd3a4a7655898" + integrity sha1-dfFmQrSA8YenEcgUFh/TpKdlWJg= + dependencies: + binary-extensions "^1.0.0" + +is-boolean-object@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/is-boolean-object/-/is-boolean-object-1.1.1.tgz#3c0878f035cb821228d350d2e1e36719716a3de8" + integrity sha512-bXdQWkECBUIAcCkeH1unwJLIpZYaa5VvuygSyS/c2lf719mTKZDU5UdDRlpd01UjADgmW8RfqaP+mRaVPdr/Ng== + dependencies: + call-bind "^1.0.2" + +is-buffer@^1.1.5: + version "1.1.6" + resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" + integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== + +is-callable@^1.1.4, is-callable@^1.2.3: + version "1.2.3" + resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.3.tgz#8b1e0500b73a1d76c70487636f368e519de8db8e" + integrity sha512-J1DcMe8UYTBSrKezuIUTUwjXsho29693unXM2YhJUTR2txK/eG47bvNa/wipPFmZFgr/N6f1GA66dv0mEyTIyQ== + +is-ci@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-2.0.0.tgz#6bc6334181810e04b5c22b3d589fdca55026404c" + integrity sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w== + dependencies: + ci-info "^2.0.0" + +is-ci@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-3.0.0.tgz#c7e7be3c9d8eef7d0fa144390bd1e4b88dc4c994" + integrity sha512-kDXyttuLeslKAHYL/K28F2YkM3x5jvFPEw3yXbRptXydjD9rpLEz+C5K5iutY9ZiUu6AP41JdvRQwF4Iqs4ZCQ== + dependencies: + ci-info "^3.1.1" + +is-core-module@^2.2.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.5.0.tgz#f754843617c70bfd29b7bd87327400cda5c18491" + integrity sha512-TXCMSDsEHMEEZ6eCA8rwRDbLu55MRGmrctljsBX/2v1d9/GzqHOxW5c5oPSgrUt2vBFXebu9rGqckXGPWOlYpg== + dependencies: + has "^1.0.3" + +is-data-descriptor@^0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz#0b5ee648388e2c860282e793f1856fec3f301b56" + integrity sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y= + dependencies: + kind-of "^3.0.2" + +is-data-descriptor@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz#d84876321d0e7add03990406abbbbd36ba9268c7" + integrity sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ== + dependencies: + kind-of "^6.0.0" + +is-date-object@^1.0.1: + version "1.0.4" + resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.4.tgz#550cfcc03afada05eea3dd30981c7b09551f73e5" + integrity sha512-/b4ZVsG7Z5XVtIxs/h9W8nvfLgSAyKYdtGWQLbqy6jA1icmgjf8WCoTKgeS4wy5tYaPePouzFMANbnj94c2Z+A== + +is-descriptor@^0.1.0: + version "0.1.6" + resolved "https://registry.yarnpkg.com/is-descriptor/-/is-descriptor-0.1.6.tgz#366d8240dde487ca51823b1ab9f07a10a78251ca" + integrity sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg== + dependencies: + is-accessor-descriptor "^0.1.6" + is-data-descriptor "^0.1.4" + kind-of "^5.0.0" + +is-descriptor@^1.0.0, is-descriptor@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-descriptor/-/is-descriptor-1.0.2.tgz#3b159746a66604b04f8c81524ba365c5f14d86ec" + integrity sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg== + dependencies: + is-accessor-descriptor "^1.0.0" + is-data-descriptor "^1.0.0" + kind-of "^6.0.2" + +is-extendable@^0.1.0, is-extendable@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/is-extendable/-/is-extendable-0.1.1.tgz#62b110e289a471418e3ec36a617d472e301dfc89" + integrity sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik= + +is-extendable@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-extendable/-/is-extendable-1.0.1.tgz#a7470f9e426733d81bd81e1155264e3a3507cab4" + integrity sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA== + dependencies: + is-plain-object "^2.0.4" + +is-extglob@^2.1.0, is-extglob@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" + integrity sha1-qIwCU1eR8C7TfHahueqXc8gz+MI= + +is-fullwidth-code-point@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb" + integrity sha1-754xOG8DGn8NZDr4L95QxFfvAMs= + dependencies: + number-is-nan "^1.0.0" + +is-fullwidth-code-point@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz#a3b30a5c4f199183167aaab93beefae3ddfb654f" + integrity sha1-o7MKXE8ZkYMWeqq5O+764937ZU8= + +is-fullwidth-code-point@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" + integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== + +is-generator-fn@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/is-generator-fn/-/is-generator-fn-2.1.0.tgz#7d140adc389aaf3011a8f2a2a4cfa6faadffb118" + integrity sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ== + +is-glob@^3.0.0, is-glob@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-3.1.0.tgz#7ba5ae24217804ac70707b96922567486cc3e84a" + integrity sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo= + dependencies: + is-extglob "^2.1.0" + +is-glob@^4.0.0, is-glob@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.1.tgz#7567dbe9f2f5e2467bc77ab83c4a29482407a5dc" + integrity sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg== + dependencies: + is-extglob "^2.1.1" + +is-lambda@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-lambda/-/is-lambda-1.0.1.tgz#3d9877899e6a53efc0160504cde15f82e6f061d5" + integrity sha1-PZh3iZ5qU+/AFgUEzeFfgubwYdU= + +is-negated-glob@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-negated-glob/-/is-negated-glob-1.0.0.tgz#6910bca5da8c95e784b5751b976cf5a10fee36d2" + integrity sha1-aRC8pdqMleeEtXUbl2z1oQ/uNtI= + +is-negative-zero@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.1.tgz#3de746c18dda2319241a53675908d8f766f11c24" + integrity sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w== + +is-number-object@^1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/is-number-object/-/is-number-object-1.0.5.tgz#6edfaeed7950cff19afedce9fbfca9ee6dd289eb" + integrity sha512-RU0lI/n95pMoUKu9v1BZP5MBcZuNSVJkMkAG2dJqC4z2GlkGUNeH68SuHuBKBD/XFe+LHZ+f9BKkLET60Niedw== + +is-number@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-number/-/is-number-3.0.0.tgz#24fd6201a4782cf50561c810276afc7d12d71195" + integrity sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU= + dependencies: + kind-of "^3.0.2" + +is-number@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/is-number/-/is-number-4.0.0.tgz#0026e37f5454d73e356dfe6564699867c6a7f0ff" + integrity sha512-rSklcAIlf1OmFdyAqbnWTLVelsQ58uvZ66S/ZyawjWqIviTWCjg2PzVGw8WUA+nNuPTqb4wgA+NszrJ+08LlgQ== + +is-number@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" + integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== + +is-obj@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/is-obj/-/is-obj-2.0.0.tgz#473fb05d973705e3fd9620545018ca8e22ef4982" + integrity sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w== + +is-path-cwd@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/is-path-cwd/-/is-path-cwd-2.2.0.tgz#67d43b82664a7b5191fd9119127eb300048a9fdb" + integrity sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ== + +is-path-inside@^3.0.2: + version "3.0.3" + resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" + integrity sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ== + +is-plain-obj@^1.0.0, is-plain-obj@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-1.1.0.tgz#71a50c8429dfca773c92a390a4a03b39fcd51d3e" + integrity sha1-caUMhCnfync8kqOQpKA7OfzVHT4= + +is-plain-obj@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-2.1.0.tgz#45e42e37fccf1f40da8e5f76ee21515840c09287" + integrity sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA== + +is-plain-object@^2.0.1, is-plain-object@^2.0.3, is-plain-object@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-2.0.4.tgz#2c163b3fafb1b606d9d17928f05c2a1c38e07677" + integrity sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og== + dependencies: + isobject "^3.0.1" + +is-plain-object@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-5.0.0.tgz#4427f50ab3429e9025ea7d52e9043a9ef4159344" + integrity sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q== + +is-potential-custom-element-name@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz#171ed6f19e3ac554394edf78caa05784a45bebb5" + integrity sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ== + +is-promise@^2.2.2: + version "2.2.2" + resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.2.2.tgz#39ab959ccbf9a774cf079f7b40c7a26f763135f1" + integrity sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ== + +is-regex@^1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.3.tgz#d029f9aff6448b93ebbe3f33dac71511fdcbef9f" + integrity sha512-qSVXFz28HM7y+IWX6vLCsexdlvzT1PJNFSBuaQLQ5o0IEw8UDYW6/2+eCMVyIsbM8CNLX2a/QWmSpyxYEHY7CQ== + dependencies: + call-bind "^1.0.2" + has-symbols "^1.0.2" + +is-relative@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-relative/-/is-relative-1.0.0.tgz#a1bb6935ce8c5dba1e8b9754b9b2dcc020e2260d" + integrity sha512-Kw/ReK0iqwKeu0MITLFuj0jbPAmEiOsIwyIXvvbfa6QfmN9pkD1M+8pdk7Rl/dTKbH34/XBFMbgD4iMJhLQbGA== + dependencies: + is-unc-path "^1.0.0" + +is-ssh@^1.3.0: + version "1.3.3" + resolved "https://registry.yarnpkg.com/is-ssh/-/is-ssh-1.3.3.tgz#7f133285ccd7f2c2c7fc897b771b53d95a2b2c7e" + integrity sha512-NKzJmQzJfEEma3w5cJNcUMxoXfDjz0Zj0eyCalHn2E6VOwlzjZo0yuO2fcBSf8zhFuVCL/82/r5gRcoi6aEPVQ== + dependencies: + protocols "^1.1.0" + +is-stream@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.0.tgz#bde9c32680d6fae04129d6ac9d921ce7815f78e3" + integrity sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw== + +is-string@^1.0.5, is-string@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.6.tgz#3fe5d5992fb0d93404f32584d4b0179a71b54a5f" + integrity sha512-2gdzbKUuqtQ3lYNrUTQYoClPhm7oQu4UdpSZMp1/DGgkHBT8E2Z1l0yMdb6D4zNAxwDiMv8MdulKROJGNl0Q0w== + +is-symbol@^1.0.2, is-symbol@^1.0.3: + version "1.0.4" + resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.4.tgz#a6dac93b635b063ca6872236de88910a57af139c" + integrity sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg== + dependencies: + has-symbols "^1.0.2" + +is-text-path@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-text-path/-/is-text-path-1.0.1.tgz#4e1aa0fb51bfbcb3e92688001397202c1775b66e" + integrity sha1-Thqg+1G/vLPpJogAE5cgLBd1tm4= + dependencies: + text-extensions "^1.0.0" + +is-typedarray@^1.0.0, is-typedarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" + integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= + +is-unc-path@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-unc-path/-/is-unc-path-1.0.0.tgz#d731e8898ed090a12c352ad2eaed5095ad322c9d" + integrity sha512-mrGpVd0fs7WWLfVsStvgF6iEJnbjDFZh9/emhRDcGWTduTfNHd9CHeUwH3gYIjdbwo4On6hunkztwOaAw0yllQ== + dependencies: + unc-path-regex "^0.1.2" + +is-utf8@^0.2.0, is-utf8@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72" + integrity sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI= + +is-valid-glob@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-valid-glob/-/is-valid-glob-1.0.0.tgz#29bf3eff701be2d4d315dbacc39bc39fe8f601aa" + integrity sha1-Kb8+/3Ab4tTTFdusw5vDn+j2Aao= + +is-windows@^1.0.1, is-windows@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d" + integrity sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA== + +isarray@1.0.0, isarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= + +isexe@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" + integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA= + +isobject@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/isobject/-/isobject-2.1.0.tgz#f065561096a3f1da2ef46272f815c840d87e0c89" + integrity sha1-8GVWEJaj8dou9GJy+BXIQNh+DIk= + dependencies: + isarray "1.0.0" + +isobject@^3.0.0, isobject@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" + integrity sha1-TkMekrEalzFjaqH5yNHMvP2reN8= + +isstream@~0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" + integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo= + +istanbul-lib-coverage@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/istanbul-lib-coverage/-/istanbul-lib-coverage-3.0.0.tgz#f5944a37c70b550b02a78a5c3b2055b280cec8ec" + integrity sha512-UiUIqxMgRDET6eR+o5HbfRYP1l0hqkWOs7vNxC/mggutCMUIhWMm8gAHb8tHlyfD3/l6rlgNA5cKdDzEAf6hEg== + +istanbul-lib-instrument@^4.0.0, istanbul-lib-instrument@^4.0.3: + version "4.0.3" + resolved "https://registry.yarnpkg.com/istanbul-lib-instrument/-/istanbul-lib-instrument-4.0.3.tgz#873c6fff897450118222774696a3f28902d77c1d" + integrity sha512-BXgQl9kf4WTCPCCpmFGoJkz/+uhvm7h7PFKUYxh7qarQd3ER33vHG//qaE8eN25l07YqZPpHXU9I09l/RD5aGQ== + dependencies: + "@babel/core" "^7.7.5" + "@istanbuljs/schema" "^0.1.2" + istanbul-lib-coverage "^3.0.0" + semver "^6.3.0" + +istanbul-lib-report@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/istanbul-lib-report/-/istanbul-lib-report-3.0.0.tgz#7518fe52ea44de372f460a76b5ecda9ffb73d8a6" + integrity sha512-wcdi+uAKzfiGT2abPpKZ0hSU1rGQjUQnLvtY5MpQ7QCTahD3VODhcu4wcfY1YtkGaDD5yuydOLINXsfbus9ROw== + dependencies: + istanbul-lib-coverage "^3.0.0" + make-dir "^3.0.0" + supports-color "^7.1.0" + +istanbul-lib-source-maps@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.0.tgz#75743ce6d96bb86dc7ee4352cf6366a23f0b1ad9" + integrity sha512-c16LpFRkR8vQXyHZ5nLpY35JZtzj1PQY1iZmesUbf1FZHbIupcWfjgOXBY9YHkLEQ6puz1u4Dgj6qmU/DisrZg== + dependencies: + debug "^4.1.1" + istanbul-lib-coverage "^3.0.0" + source-map "^0.6.1" + +istanbul-reports@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/istanbul-reports/-/istanbul-reports-3.0.2.tgz#d593210e5000683750cb09fc0644e4b6e27fd53b" + integrity sha512-9tZvz7AiR3PEDNGiV9vIouQ/EAcqMXFmkcA1CDFTwOB98OZVDL0PH9glHotf5Ugp6GCOTypfzGWI/OqjWNCRUw== + dependencies: + html-escaper "^2.0.0" + istanbul-lib-report "^3.0.0" + +ix@4.4.1: + version "4.4.1" + resolved "https://registry.yarnpkg.com/ix/-/ix-4.4.1.tgz#8ec5f4f420c504a9906ffc2e2234f50147b9488a" + integrity sha512-Jsl7cUf7CA1MkznzAuVy4K6V1Zsfx+EAh0ZgiGhGAADaEGKiMV+sJx8Qe4hx0CsyI475Yt3ppoRS8M8oOueqlA== + dependencies: + "@types/node" "^13.7.4" + tslib "^2.3.0" + +jest-changed-files@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-changed-files/-/jest-changed-files-27.0.6.tgz#bed6183fcdea8a285482e3b50a9a7712d49a7a8b" + integrity sha512-BuL/ZDauaq5dumYh5y20sn4IISnf1P9A0TDswTxUi84ORGtVa86ApuBHqICL0vepqAnZiY6a7xeSPWv2/yy4eA== + dependencies: + "@jest/types" "^27.0.6" + execa "^5.0.0" + throat "^6.0.1" + +jest-circus@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-circus/-/jest-circus-27.0.6.tgz#dd4df17c4697db6a2c232aaad4e9cec666926668" + integrity sha512-OJlsz6BBeX9qR+7O9lXefWoc2m9ZqcZ5Ohlzz0pTEAG4xMiZUJoacY8f4YDHxgk0oKYxj277AfOk9w6hZYvi1Q== + dependencies: + "@jest/environment" "^27.0.6" + "@jest/test-result" "^27.0.6" + "@jest/types" "^27.0.6" + "@types/node" "*" + chalk "^4.0.0" + co "^4.6.0" + dedent "^0.7.0" + expect "^27.0.6" + is-generator-fn "^2.0.0" + jest-each "^27.0.6" + jest-matcher-utils "^27.0.6" + jest-message-util "^27.0.6" + jest-runtime "^27.0.6" + jest-snapshot "^27.0.6" + jest-util "^27.0.6" + pretty-format "^27.0.6" + slash "^3.0.0" + stack-utils "^2.0.3" + throat "^6.0.1" + +jest-cli@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-cli/-/jest-cli-27.0.6.tgz#d021e5f4d86d6a212450d4c7b86cb219f1e6864f" + integrity sha512-qUUVlGb9fdKir3RDE+B10ULI+LQrz+MCflEH2UJyoUjoHHCbxDrMxSzjQAPUMsic4SncI62ofYCcAvW6+6rhhg== + dependencies: + "@jest/core" "^27.0.6" + "@jest/test-result" "^27.0.6" + "@jest/types" "^27.0.6" + chalk "^4.0.0" + exit "^0.1.2" + graceful-fs "^4.2.4" + import-local "^3.0.2" + jest-config "^27.0.6" + jest-util "^27.0.6" + jest-validate "^27.0.6" + prompts "^2.0.1" + yargs "^16.0.3" + +jest-config@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-config/-/jest-config-27.0.6.tgz#119fb10f149ba63d9c50621baa4f1f179500277f" + integrity sha512-JZRR3I1Plr2YxPBhgqRspDE2S5zprbga3swYNrvY3HfQGu7p/GjyLOqwrYad97tX3U3mzT53TPHVmozacfP/3w== + dependencies: + "@babel/core" "^7.1.0" + "@jest/test-sequencer" "^27.0.6" + "@jest/types" "^27.0.6" + babel-jest "^27.0.6" + chalk "^4.0.0" + deepmerge "^4.2.2" + glob "^7.1.1" + graceful-fs "^4.2.4" + is-ci "^3.0.0" + jest-circus "^27.0.6" + jest-environment-jsdom "^27.0.6" + jest-environment-node "^27.0.6" + jest-get-type "^27.0.6" + jest-jasmine2 "^27.0.6" + jest-regex-util "^27.0.6" + jest-resolve "^27.0.6" + jest-runner "^27.0.6" + jest-util "^27.0.6" + jest-validate "^27.0.6" + micromatch "^4.0.4" + pretty-format "^27.0.6" + +jest-diff@^26.0.0: + version "26.6.2" + resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-26.6.2.tgz#1aa7468b52c3a68d7d5c5fdcdfcd5e49bd164394" + integrity sha512-6m+9Z3Gv9wN0WFVasqjCL/06+EFCMTqDEUl/b87HYK2rAPTyfz4ZIuSlPhY51PIQRWx5TaxeF1qmXKe9gfN3sA== + dependencies: + chalk "^4.0.0" + diff-sequences "^26.6.2" + jest-get-type "^26.3.0" + pretty-format "^26.6.2" + +jest-diff@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-27.0.6.tgz#4a7a19ee6f04ad70e0e3388f35829394a44c7b5e" + integrity sha512-Z1mqgkTCSYaFgwTlP/NUiRzdqgxmmhzHY1Tq17zL94morOHfHu3K4bgSgl+CR4GLhpV8VxkuOYuIWnQ9LnFqmg== + dependencies: + chalk "^4.0.0" + diff-sequences "^27.0.6" + jest-get-type "^27.0.6" + pretty-format "^27.0.6" + +jest-docblock@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-docblock/-/jest-docblock-27.0.6.tgz#cc78266acf7fe693ca462cbbda0ea4e639e4e5f3" + integrity sha512-Fid6dPcjwepTFraz0YxIMCi7dejjJ/KL9FBjPYhBp4Sv1Y9PdhImlKZqYU555BlN4TQKaTc+F2Av1z+anVyGkA== + dependencies: + detect-newline "^3.0.0" + +jest-each@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-each/-/jest-each-27.0.6.tgz#cee117071b04060158dc8d9a66dc50ad40ef453b" + integrity sha512-m6yKcV3bkSWrUIjxkE9OC0mhBZZdhovIW5ergBYirqnkLXkyEn3oUUF/QZgyecA1cF1QFyTE8bRRl8Tfg1pfLA== + dependencies: + "@jest/types" "^27.0.6" + chalk "^4.0.0" + jest-get-type "^27.0.6" + jest-util "^27.0.6" + pretty-format "^27.0.6" + +jest-environment-jsdom@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-environment-jsdom/-/jest-environment-jsdom-27.0.6.tgz#f66426c4c9950807d0a9f209c590ce544f73291f" + integrity sha512-FvetXg7lnXL9+78H+xUAsra3IeZRTiegA3An01cWeXBspKXUhAwMM9ycIJ4yBaR0L7HkoMPaZsozCLHh4T8fuw== + dependencies: + "@jest/environment" "^27.0.6" + "@jest/fake-timers" "^27.0.6" + "@jest/types" "^27.0.6" + "@types/node" "*" + jest-mock "^27.0.6" + jest-util "^27.0.6" + jsdom "^16.6.0" + +jest-environment-node@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-environment-node/-/jest-environment-node-27.0.6.tgz#a6699b7ceb52e8d68138b9808b0c404e505f3e07" + integrity sha512-+Vi6yLrPg/qC81jfXx3IBlVnDTI6kmRr08iVa2hFCWmJt4zha0XW7ucQltCAPhSR0FEKEoJ3i+W4E6T0s9is0w== + dependencies: + "@jest/environment" "^27.0.6" + "@jest/fake-timers" "^27.0.6" + "@jest/types" "^27.0.6" + "@types/node" "*" + jest-mock "^27.0.6" + jest-util "^27.0.6" + +jest-get-type@^26.3.0: + version "26.3.0" + resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-26.3.0.tgz#e97dc3c3f53c2b406ca7afaed4493b1d099199e0" + integrity sha512-TpfaviN1R2pQWkIihlfEanwOXK0zcxrKEE4MlU6Tn7keoXdN6/3gK/xl0yEh8DOunn5pOVGKf8hB4R9gVh04ig== + +jest-get-type@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-27.0.6.tgz#0eb5c7f755854279ce9b68a9f1a4122f69047cfe" + integrity sha512-XTkK5exIeUbbveehcSR8w0bhH+c0yloW/Wpl+9vZrjzztCPWrxhHwkIFpZzCt71oRBsgxmuUfxEqOYoZI2macg== + +jest-haste-map@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-haste-map/-/jest-haste-map-27.0.6.tgz#4683a4e68f6ecaa74231679dca237279562c8dc7" + integrity sha512-4ldjPXX9h8doB2JlRzg9oAZ2p6/GpQUNAeiYXqcpmrKbP0Qev0wdZlxSMOmz8mPOEnt4h6qIzXFLDi8RScX/1w== + dependencies: + "@jest/types" "^27.0.6" + "@types/graceful-fs" "^4.1.2" + "@types/node" "*" + anymatch "^3.0.3" + fb-watchman "^2.0.0" + graceful-fs "^4.2.4" + jest-regex-util "^27.0.6" + jest-serializer "^27.0.6" + jest-util "^27.0.6" + jest-worker "^27.0.6" + micromatch "^4.0.4" + walker "^1.0.7" + optionalDependencies: + fsevents "^2.3.2" + +jest-jasmine2@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-jasmine2/-/jest-jasmine2-27.0.6.tgz#fd509a9ed3d92bd6edb68a779f4738b100655b37" + integrity sha512-cjpH2sBy+t6dvCeKBsHpW41mjHzXgsavaFMp+VWRf0eR4EW8xASk1acqmljFtK2DgyIECMv2yCdY41r2l1+4iA== + dependencies: + "@babel/traverse" "^7.1.0" + "@jest/environment" "^27.0.6" + "@jest/source-map" "^27.0.6" + "@jest/test-result" "^27.0.6" + "@jest/types" "^27.0.6" + "@types/node" "*" + chalk "^4.0.0" + co "^4.6.0" + expect "^27.0.6" + is-generator-fn "^2.0.0" + jest-each "^27.0.6" + jest-matcher-utils "^27.0.6" + jest-message-util "^27.0.6" + jest-runtime "^27.0.6" + jest-snapshot "^27.0.6" + jest-util "^27.0.6" + pretty-format "^27.0.6" + throat "^6.0.1" + +jest-leak-detector@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-leak-detector/-/jest-leak-detector-27.0.6.tgz#545854275f85450d4ef4b8fe305ca2a26450450f" + integrity sha512-2/d6n2wlH5zEcdctX4zdbgX8oM61tb67PQt4Xh8JFAIy6LRKUnX528HulkaG6nD5qDl5vRV1NXejCe1XRCH5gQ== + dependencies: + jest-get-type "^27.0.6" + pretty-format "^27.0.6" + +jest-matcher-utils@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-27.0.6.tgz#2a8da1e86c620b39459f4352eaa255f0d43e39a9" + integrity sha512-OFgF2VCQx9vdPSYTHWJ9MzFCehs20TsyFi6bIHbk5V1u52zJOnvF0Y/65z3GLZHKRuTgVPY4Z6LVePNahaQ+tA== + dependencies: + chalk "^4.0.0" + jest-diff "^27.0.6" + jest-get-type "^27.0.6" + pretty-format "^27.0.6" + +jest-message-util@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-27.0.6.tgz#158bcdf4785706492d164a39abca6a14da5ab8b5" + integrity sha512-rBxIs2XK7rGy+zGxgi+UJKP6WqQ+KrBbD1YMj517HYN3v2BG66t3Xan3FWqYHKZwjdB700KiAJ+iES9a0M+ixw== + dependencies: + "@babel/code-frame" "^7.12.13" + "@jest/types" "^27.0.6" + "@types/stack-utils" "^2.0.0" + chalk "^4.0.0" + graceful-fs "^4.2.4" + micromatch "^4.0.4" + pretty-format "^27.0.6" + slash "^3.0.0" + stack-utils "^2.0.3" + +jest-mock@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-27.0.6.tgz#0efdd40851398307ba16778728f6d34d583e3467" + integrity sha512-lzBETUoK8cSxts2NYXSBWT+EJNzmUVtVVwS1sU9GwE1DLCfGsngg+ZVSIe0yd0ZSm+y791esiuo+WSwpXJQ5Bw== + dependencies: + "@jest/types" "^27.0.6" + "@types/node" "*" + +jest-pnp-resolver@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/jest-pnp-resolver/-/jest-pnp-resolver-1.2.2.tgz#b704ac0ae028a89108a4d040b3f919dfddc8e33c" + integrity sha512-olV41bKSMm8BdnuMsewT4jqlZ8+3TCARAXjZGT9jcoSnrfUnRCqnMoF9XEeoWjbzObpqF9dRhHQj0Xb9QdF6/w== + +jest-regex-util@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-regex-util/-/jest-regex-util-27.0.6.tgz#02e112082935ae949ce5d13b2675db3d8c87d9c5" + integrity sha512-SUhPzBsGa1IKm8hx2F4NfTGGp+r7BXJ4CulsZ1k2kI+mGLG+lxGrs76veN2LF/aUdGosJBzKgXmNCw+BzFqBDQ== + +jest-resolve-dependencies@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-resolve-dependencies/-/jest-resolve-dependencies-27.0.6.tgz#3e619e0ef391c3ecfcf6ef4056207a3d2be3269f" + integrity sha512-mg9x9DS3BPAREWKCAoyg3QucCr0n6S8HEEsqRCKSPjPcu9HzRILzhdzY3imsLoZWeosEbJZz6TKasveczzpJZA== + dependencies: + "@jest/types" "^27.0.6" + jest-regex-util "^27.0.6" + jest-snapshot "^27.0.6" + +jest-resolve@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-resolve/-/jest-resolve-27.0.6.tgz#e90f436dd4f8fbf53f58a91c42344864f8e55bff" + integrity sha512-yKmIgw2LgTh7uAJtzv8UFHGF7Dm7XfvOe/LQ3Txv101fLM8cx2h1QVwtSJ51Q/SCxpIiKfVn6G2jYYMDNHZteA== + dependencies: + "@jest/types" "^27.0.6" + chalk "^4.0.0" + escalade "^3.1.1" + graceful-fs "^4.2.4" + jest-pnp-resolver "^1.2.2" + jest-util "^27.0.6" + jest-validate "^27.0.6" + resolve "^1.20.0" + slash "^3.0.0" + +jest-runner@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-runner/-/jest-runner-27.0.6.tgz#1325f45055539222bbc7256a6976e993ad2f9520" + integrity sha512-W3Bz5qAgaSChuivLn+nKOgjqNxM7O/9JOJoKDCqThPIg2sH/d4A/lzyiaFgnb9V1/w29Le11NpzTJSzga1vyYQ== + dependencies: + "@jest/console" "^27.0.6" + "@jest/environment" "^27.0.6" + "@jest/test-result" "^27.0.6" + "@jest/transform" "^27.0.6" + "@jest/types" "^27.0.6" + "@types/node" "*" + chalk "^4.0.0" + emittery "^0.8.1" + exit "^0.1.2" + graceful-fs "^4.2.4" + jest-docblock "^27.0.6" + jest-environment-jsdom "^27.0.6" + jest-environment-node "^27.0.6" + jest-haste-map "^27.0.6" + jest-leak-detector "^27.0.6" + jest-message-util "^27.0.6" + jest-resolve "^27.0.6" + jest-runtime "^27.0.6" + jest-util "^27.0.6" + jest-worker "^27.0.6" + source-map-support "^0.5.6" + throat "^6.0.1" + +jest-runtime@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-runtime/-/jest-runtime-27.0.6.tgz#45877cfcd386afdd4f317def551fc369794c27c9" + integrity sha512-BhvHLRVfKibYyqqEFkybsznKwhrsu7AWx2F3y9G9L95VSIN3/ZZ9vBpm/XCS2bS+BWz3sSeNGLzI3TVQ0uL85Q== + dependencies: + "@jest/console" "^27.0.6" + "@jest/environment" "^27.0.6" + "@jest/fake-timers" "^27.0.6" + "@jest/globals" "^27.0.6" + "@jest/source-map" "^27.0.6" + "@jest/test-result" "^27.0.6" + "@jest/transform" "^27.0.6" + "@jest/types" "^27.0.6" + "@types/yargs" "^16.0.0" + chalk "^4.0.0" + cjs-module-lexer "^1.0.0" + collect-v8-coverage "^1.0.0" + exit "^0.1.2" + glob "^7.1.3" + graceful-fs "^4.2.4" + jest-haste-map "^27.0.6" + jest-message-util "^27.0.6" + jest-mock "^27.0.6" + jest-regex-util "^27.0.6" + jest-resolve "^27.0.6" + jest-snapshot "^27.0.6" + jest-util "^27.0.6" + jest-validate "^27.0.6" + slash "^3.0.0" + strip-bom "^4.0.0" + yargs "^16.0.3" + +jest-serializer@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-serializer/-/jest-serializer-27.0.6.tgz#93a6c74e0132b81a2d54623251c46c498bb5bec1" + integrity sha512-PtGdVK9EGC7dsaziskfqaAPib6wTViY3G8E5wz9tLVPhHyiDNTZn/xjZ4khAw+09QkoOVpn7vF5nPSN6dtBexA== + dependencies: + "@types/node" "*" + graceful-fs "^4.2.4" + +jest-silent-reporter@0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/jest-silent-reporter/-/jest-silent-reporter-0.5.0.tgz#5fd8ccd61665227e3bf19d908b7350719d06ff38" + integrity sha512-epdLt8Oj0a1AyRiR6F8zx/1SVT1Mi7VU3y4wB2uOBHs/ohIquC7v2eeja7UN54uRPyHInIKWdL+RdG228n5pJQ== + dependencies: + chalk "^4.0.0" + jest-util "^26.0.0" + +jest-snapshot@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-snapshot/-/jest-snapshot-27.0.6.tgz#f4e6b208bd2e92e888344d78f0f650bcff05a4bf" + integrity sha512-NTHaz8He+ATUagUgE7C/UtFcRoHqR2Gc+KDfhQIyx+VFgwbeEMjeP+ILpUTLosZn/ZtbNdCF5LkVnN/l+V751A== + dependencies: + "@babel/core" "^7.7.2" + "@babel/generator" "^7.7.2" + "@babel/parser" "^7.7.2" + "@babel/plugin-syntax-typescript" "^7.7.2" + "@babel/traverse" "^7.7.2" + "@babel/types" "^7.0.0" + "@jest/transform" "^27.0.6" + "@jest/types" "^27.0.6" + "@types/babel__traverse" "^7.0.4" + "@types/prettier" "^2.1.5" + babel-preset-current-node-syntax "^1.0.0" + chalk "^4.0.0" + expect "^27.0.6" + graceful-fs "^4.2.4" + jest-diff "^27.0.6" + jest-get-type "^27.0.6" + jest-haste-map "^27.0.6" + jest-matcher-utils "^27.0.6" + jest-message-util "^27.0.6" + jest-resolve "^27.0.6" + jest-util "^27.0.6" + natural-compare "^1.4.0" + pretty-format "^27.0.6" + semver "^7.3.2" + +jest-util@^26.0.0: + version "26.6.2" + resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-26.6.2.tgz#907535dbe4d5a6cb4c47ac9b926f6af29576cbc1" + integrity sha512-MDW0fKfsn0OI7MS7Euz6h8HNDXVQ0gaM9uW6RjfDmd1DAFcaxX9OqIakHIqhbnmF08Cf2DLDG+ulq8YQQ0Lp0Q== + dependencies: + "@jest/types" "^26.6.2" + "@types/node" "*" + chalk "^4.0.0" + graceful-fs "^4.2.4" + is-ci "^2.0.0" + micromatch "^4.0.2" + +jest-util@^27.0.0, jest-util@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-27.0.6.tgz#e8e04eec159de2f4d5f57f795df9cdc091e50297" + integrity sha512-1JjlaIh+C65H/F7D11GNkGDDZtDfMEM8EBXsvd+l/cxtgQ6QhxuloOaiayt89DxUvDarbVhqI98HhgrM1yliFQ== + dependencies: + "@jest/types" "^27.0.6" + "@types/node" "*" + chalk "^4.0.0" + graceful-fs "^4.2.4" + is-ci "^3.0.0" + picomatch "^2.2.3" + +jest-validate@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-validate/-/jest-validate-27.0.6.tgz#930a527c7a951927df269f43b2dc23262457e2a6" + integrity sha512-yhZZOaMH3Zg6DC83n60pLmdU1DQE46DW+KLozPiPbSbPhlXXaiUTDlhHQhHFpaqIFRrInko1FHXjTRpjWRuWfA== + dependencies: + "@jest/types" "^27.0.6" + camelcase "^6.2.0" + chalk "^4.0.0" + jest-get-type "^27.0.6" + leven "^3.1.0" + pretty-format "^27.0.6" + +jest-watcher@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-watcher/-/jest-watcher-27.0.6.tgz#89526f7f9edf1eac4e4be989bcb6dec6b8878d9c" + integrity sha512-/jIoKBhAP00/iMGnTwUBLgvxkn7vsOweDrOTSPzc7X9uOyUtJIDthQBTI1EXz90bdkrxorUZVhJwiB69gcHtYQ== + dependencies: + "@jest/test-result" "^27.0.6" + "@jest/types" "^27.0.6" + "@types/node" "*" + ansi-escapes "^4.2.1" + chalk "^4.0.0" + jest-util "^27.0.6" + string-length "^4.0.1" + +jest-worker@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-27.0.6.tgz#a5fdb1e14ad34eb228cfe162d9f729cdbfa28aed" + integrity sha512-qupxcj/dRuA3xHPMUd40gr2EaAurFbkwzOh7wfPaeE9id7hyjURRQoqNfHifHK3XjJU6YJJUQKILGUnwGPEOCA== + dependencies: + "@types/node" "*" + merge-stream "^2.0.0" + supports-color "^8.0.0" + +jest@27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/jest/-/jest-27.0.6.tgz#10517b2a628f0409087fbf473db44777d7a04505" + integrity sha512-EjV8aETrsD0wHl7CKMibKwQNQc3gIRBXlTikBmmHUeVMKaPFxdcUIBfoDqTSXDoGJIivAYGqCWVlzCSaVjPQsA== + dependencies: + "@jest/core" "^27.0.6" + import-local "^3.0.2" + jest-cli "^27.0.6" + +js-tokens@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" + integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== + +js-yaml@^3.13.1: + version "3.14.1" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.14.1.tgz#dae812fdb3825fa306609a8717383c50c36a0537" + integrity sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g== + dependencies: + argparse "^1.0.7" + esprima "^4.0.0" + +jsbn@~0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" + integrity sha1-peZUwuWi3rXyAdls77yoDA7y9RM= + +jsdom@^16.6.0: + version "16.6.0" + resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-16.6.0.tgz#f79b3786682065492a3da6a60a4695da983805ac" + integrity sha512-Ty1vmF4NHJkolaEmdjtxTfSfkdb8Ywarwf63f+F8/mDD1uLSSWDxDuMiZxiPhwunLrn9LOSVItWj4bLYsLN3Dg== + dependencies: + abab "^2.0.5" + acorn "^8.2.4" + acorn-globals "^6.0.0" + cssom "^0.4.4" + cssstyle "^2.3.0" + data-urls "^2.0.0" + decimal.js "^10.2.1" + domexception "^2.0.1" + escodegen "^2.0.0" + form-data "^3.0.0" + html-encoding-sniffer "^2.0.1" + http-proxy-agent "^4.0.1" + https-proxy-agent "^5.0.0" + is-potential-custom-element-name "^1.0.1" + nwsapi "^2.2.0" + parse5 "6.0.1" + saxes "^5.0.1" + symbol-tree "^3.2.4" + tough-cookie "^4.0.0" + w3c-hr-time "^1.0.2" + w3c-xmlserializer "^2.0.0" + webidl-conversions "^6.1.0" + whatwg-encoding "^1.0.5" + whatwg-mimetype "^2.3.0" + whatwg-url "^8.5.0" + ws "^7.4.5" + xml-name-validator "^3.0.0" + +jsesc@^2.5.1: + version "2.5.2" + resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" + integrity sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA== + +json-bignum@^0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/json-bignum/-/json-bignum-0.0.3.tgz#41163b50436c773d82424dbc20ed70db7604b8d7" + integrity sha1-QRY7UENsdz2CQk28IO1w23YEuNc= + +json-parse-better-errors@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9" + integrity sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw== + +json-parse-even-better-errors@^2.3.0: + version "2.3.1" + resolved "https://registry.yarnpkg.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d" + integrity sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w== + +json-schema-traverse@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" + integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== + +json-schema-traverse@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz#ae7bcb3656ab77a73ba5c49bf654f38e6b6860e2" + integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug== + +json-schema@0.2.3: + version "0.2.3" + resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" + integrity sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM= + +json-stable-stringify-without-jsonify@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" + integrity sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE= + +json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" + integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus= + +json2csv@^5.0.4: + version "5.0.6" + resolved "https://registry.yarnpkg.com/json2csv/-/json2csv-5.0.6.tgz#590e0e1b9579e59baa53bda0c0d840f4d8009687" + integrity sha512-0/4Lv6IenJV0qj2oBdgPIAmFiKKnh8qh7bmLFJ+/ZZHLjSeiL3fKKGX3UryvKPbxFbhV+JcYo9KUC19GJ/Z/4A== + dependencies: + commander "^6.1.0" + jsonparse "^1.3.1" + lodash.get "^4.4.2" + +json5@2.x, json5@^2.1.2, json5@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.0.tgz#2dfefe720c6ba525d9ebd909950f0515316c89a3" + integrity sha512-f+8cldu7X/y7RAJurMEJmdoKXGB/X550w2Nr3tTbezL6RwEE/iMcm+tZnXeoZtKuOq6ft8+CqzEkrIgx1fPoQA== + dependencies: + minimist "^1.2.5" + +jsonfile@^6.0.1: + version "6.1.0" + resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.1.0.tgz#bc55b2634793c679ec6403094eb13698a6ec0aae" + integrity sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ== + dependencies: + universalify "^2.0.0" + optionalDependencies: + graceful-fs "^4.1.6" + +jsonparse@^1.2.0, jsonparse@^1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" + integrity sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA= + +jsprim@^1.2.2: + version "1.4.1" + resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" + integrity sha1-MT5mvB5cwG5Di8G3SZwuXFastqI= + dependencies: + assert-plus "1.0.0" + extsprintf "1.3.0" + json-schema "0.2.3" + verror "1.10.0" + +junk@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/junk/-/junk-3.1.0.tgz#31499098d902b7e98c5d9b9c80f43457a88abfa1" + integrity sha512-pBxcB3LFc8QVgdggvZWyeys+hnrNWg4OcZIU/1X59k5jQdLBlCsYGRQaz234SqoRLTCgMH00fY0xRJH+F9METQ== + +just-debounce@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/just-debounce/-/just-debounce-1.1.0.tgz#2f81a3ad4121a76bc7cb45dbf704c0d76a8e5ddf" + integrity sha512-qpcRocdkUmf+UTNBYx5w6dexX5J31AKK1OmPwH630a83DdVVUIngk55RSAiIGpQyoH0dlr872VHfPjnQnK1qDQ== + +kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0: + version "3.2.2" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" + integrity sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ= + dependencies: + is-buffer "^1.1.5" + +kind-of@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-4.0.0.tgz#20813df3d712928b207378691a45066fae72dd57" + integrity sha1-IIE989cSkosgc3hpGkUGb65y3Vc= + dependencies: + is-buffer "^1.1.5" + +kind-of@^5.0.0, kind-of@^5.0.2: + version "5.1.0" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-5.1.0.tgz#729c91e2d857b7a419a1f9aa65685c4c33f5845d" + integrity sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw== + +kind-of@^6.0.0, kind-of@^6.0.2, kind-of@^6.0.3: + version "6.0.3" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" + integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== + +kleur@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" + integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w== + +kleur@^4.1.3: + version "4.1.4" + resolved "https://registry.yarnpkg.com/kleur/-/kleur-4.1.4.tgz#8c202987d7e577766d039a8cd461934c01cda04d" + integrity sha512-8QADVssbrFjivHWQU7KkMgptGTl6WAcSdlbBPY4uNF+mWr6DGcKrvY2w4FQJoXch7+fKMjj0dRrL75vk3k23OA== + +last-run@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/last-run/-/last-run-1.1.1.tgz#45b96942c17b1c79c772198259ba943bebf8ca5b" + integrity sha1-RblpQsF7HHnHchmCWbqUO+v4yls= + dependencies: + default-resolution "^2.0.0" + es6-weak-map "^2.0.1" + +lazystream@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/lazystream/-/lazystream-1.0.0.tgz#f6995fe0f820392f61396be89462407bb77168e4" + integrity sha1-9plf4PggOS9hOWvolGJAe7dxaOQ= + dependencies: + readable-stream "^2.0.5" + +lcid@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/lcid/-/lcid-1.0.0.tgz#308accafa0bc483a3867b4b6f2b9506251d1b835" + integrity sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU= + dependencies: + invert-kv "^1.0.0" + +lead@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/lead/-/lead-1.0.0.tgz#6f14f99a37be3a9dd784f5495690e5903466ee42" + integrity sha1-bxT5mje+Op3XhPVJVpDlkDRm7kI= + dependencies: + flush-write-stream "^1.0.2" + +lerna@4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/lerna/-/lerna-4.0.0.tgz#b139d685d50ea0ca1be87713a7c2f44a5b678e9e" + integrity sha512-DD/i1znurfOmNJb0OBw66NmNqiM8kF6uIrzrJ0wGE3VNdzeOhz9ziWLYiRaZDGGwgbcjOo6eIfcx9O5Qynz+kg== + dependencies: + "@lerna/add" "4.0.0" + "@lerna/bootstrap" "4.0.0" + "@lerna/changed" "4.0.0" + "@lerna/clean" "4.0.0" + "@lerna/cli" "4.0.0" + "@lerna/create" "4.0.0" + "@lerna/diff" "4.0.0" + "@lerna/exec" "4.0.0" + "@lerna/import" "4.0.0" + "@lerna/info" "4.0.0" + "@lerna/init" "4.0.0" + "@lerna/link" "4.0.0" + "@lerna/list" "4.0.0" + "@lerna/publish" "4.0.0" + "@lerna/run" "4.0.0" + "@lerna/version" "4.0.0" + import-local "^3.0.2" + npmlog "^4.1.2" + +leven@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/leven/-/leven-3.1.0.tgz#77891de834064cccba82ae7842bb6b14a13ed7f2" + integrity sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A== + +levn@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade" + integrity sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ== + dependencies: + prelude-ls "^1.2.1" + type-check "~0.4.0" + +levn@~0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/levn/-/levn-0.3.0.tgz#3b09924edf9f083c0490fdd4c0bc4421e04764ee" + integrity sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4= + dependencies: + prelude-ls "~1.1.2" + type-check "~0.3.2" + +libnpmaccess@^4.0.1: + version "4.0.3" + resolved "https://registry.yarnpkg.com/libnpmaccess/-/libnpmaccess-4.0.3.tgz#dfb0e5b0a53c315a2610d300e46b4ddeb66e7eec" + integrity sha512-sPeTSNImksm8O2b6/pf3ikv4N567ERYEpeKRPSmqlNt1dTZbvgpJIzg5vAhXHpw2ISBsELFRelk0jEahj1c6nQ== + dependencies: + aproba "^2.0.0" + minipass "^3.1.1" + npm-package-arg "^8.1.2" + npm-registry-fetch "^11.0.0" + +libnpmpublish@^4.0.0: + version "4.0.2" + resolved "https://registry.yarnpkg.com/libnpmpublish/-/libnpmpublish-4.0.2.tgz#be77e8bf5956131bcb45e3caa6b96a842dec0794" + integrity sha512-+AD7A2zbVeGRCFI2aO//oUmapCwy7GHqPXFJh3qpToSRNU+tXKJ2YFUgjt04LPPAf2dlEH95s6EhIHM1J7bmOw== + dependencies: + normalize-package-data "^3.0.2" + npm-package-arg "^8.1.2" + npm-registry-fetch "^11.0.0" + semver "^7.1.3" + ssri "^8.0.1" + +liftoff@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/liftoff/-/liftoff-3.1.0.tgz#c9ba6081f908670607ee79062d700df062c52ed3" + integrity sha512-DlIPlJUkCV0Ips2zf2pJP0unEoT1kwYhiiPUGF3s/jtxTCjziNLoiVVh+jqWOWeFi6mmwQ5fNxvAUyPad4Dfog== + dependencies: + extend "^3.0.0" + findup-sync "^3.0.0" + fined "^1.0.1" + flagged-respawn "^1.0.0" + is-plain-object "^2.0.4" + object.map "^1.0.0" + rechoir "^0.6.2" + resolve "^1.1.7" + +lines-and-columns@^1.1.6: + version "1.1.6" + resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00" + integrity sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA= + +load-json-file@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-1.1.0.tgz#956905708d58b4bab4c2261b04f59f31c99374c0" + integrity sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA= + dependencies: + graceful-fs "^4.1.2" + parse-json "^2.2.0" + pify "^2.0.0" + pinkie-promise "^2.0.0" + strip-bom "^2.0.0" + +load-json-file@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-4.0.0.tgz#2f5f45ab91e33216234fd53adab668eb4ec0993b" + integrity sha1-L19Fq5HjMhYjT9U62rZo607AmTs= + dependencies: + graceful-fs "^4.1.2" + parse-json "^4.0.0" + pify "^3.0.0" + strip-bom "^3.0.0" + +load-json-file@^6.2.0: + version "6.2.0" + resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-6.2.0.tgz#5c7770b42cafa97074ca2848707c61662f4251a1" + integrity sha512-gUD/epcRms75Cw8RT1pUdHugZYM5ce64ucs2GEISABwkRsOQr0q2wm/MV2TKThycIe5e0ytRweW2RZxclogCdQ== + dependencies: + graceful-fs "^4.1.15" + parse-json "^5.0.0" + strip-bom "^4.0.0" + type-fest "^0.6.0" + +locate-path@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-2.0.0.tgz#2b568b265eec944c6d9c0de9c3dbbbca0354cd8e" + integrity sha1-K1aLJl7slExtnA3pw9u7ygNUzY4= + dependencies: + p-locate "^2.0.0" + path-exists "^3.0.0" + +locate-path@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-5.0.0.tgz#1afba396afd676a6d42504d0a67a3a7eb9f62aa0" + integrity sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g== + dependencies: + p-locate "^4.1.0" + +locate-path@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286" + integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw== + dependencies: + p-locate "^5.0.0" + +lodash._reinterpolate@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz#0ccf2d89166af03b3663c796538b75ac6e114d9d" + integrity sha1-DM8tiRZq8Ds2Y8eWU4t1rG4RTZ0= + +lodash.camelcase@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6" + integrity sha1-soqmKIorn8ZRA1x3EfZathkDMaY= + +lodash.clonedeep@^4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef" + integrity sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8= + +lodash.get@^4.4.2: + version "4.4.2" + resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99" + integrity sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk= + +lodash.ismatch@^4.4.0: + version "4.4.0" + resolved "https://registry.yarnpkg.com/lodash.ismatch/-/lodash.ismatch-4.4.0.tgz#756cb5150ca3ba6f11085a78849645f188f85f37" + integrity sha1-dWy1FQyjum8RCFp4hJZF8Yj4Xzc= + +lodash.merge@^4.6.2: + version "4.6.2" + resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" + integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== + +lodash.template@^4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/lodash.template/-/lodash.template-4.5.0.tgz#f976195cf3f347d0d5f52483569fe8031ccce8ab" + integrity sha512-84vYFxIkmidUiFxidA/KjjH9pAycqW+h980j7Fuz5qxRtO9pgB7MDFTdys1N7A5mcucRiDyEq4fusljItR1T/A== + dependencies: + lodash._reinterpolate "^3.0.0" + lodash.templatesettings "^4.0.0" + +lodash.templatesettings@^4.0.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/lodash.templatesettings/-/lodash.templatesettings-4.2.0.tgz#e481310f049d3cf6d47e912ad09313b154f0fb33" + integrity sha512-stgLz+i3Aa9mZgnjr/O+v9ruKZsPsndy7qPZOchbqk2cnTU1ZaldKK+v7m54WoKIyxiuMZTKT2H81F8BeAc3ZQ== + dependencies: + lodash._reinterpolate "^3.0.0" + +lodash.truncate@^4.4.2: + version "4.4.2" + resolved "https://registry.yarnpkg.com/lodash.truncate/-/lodash.truncate-4.4.2.tgz#5a350da0b1113b837ecfffd5812cbe58d6eae193" + integrity sha1-WjUNoLERO4N+z//VgSy+WNbq4ZM= + +lodash@4.x, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.4, lodash@^4.7.0: + version "4.17.21" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" + integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== + +log-update@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/log-update/-/log-update-4.0.0.tgz#589ecd352471f2a1c0c570287543a64dfd20e0a1" + integrity sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg== + dependencies: + ansi-escapes "^4.3.0" + cli-cursor "^3.1.0" + slice-ansi "^4.0.0" + wrap-ansi "^6.2.0" + +lru-cache@^5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920" + integrity sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w== + dependencies: + yallist "^3.0.2" + +lru-cache@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-6.0.0.tgz#6d6fe6570ebd96aaf90fcad1dafa3b2566db3a94" + integrity sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA== + dependencies: + yallist "^4.0.0" + +lru-queue@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/lru-queue/-/lru-queue-0.1.0.tgz#2738bd9f0d3cf4f84490c5736c48699ac632cda3" + integrity sha1-Jzi9nw089PhEkMVzbEhpmsYyzaM= + dependencies: + es5-ext "~0.10.2" + +lunr@^2.3.9: + version "2.3.9" + resolved "https://registry.yarnpkg.com/lunr/-/lunr-2.3.9.tgz#18b123142832337dd6e964df1a5a7707b25d35e1" + integrity sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow== + +make-dir@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-2.1.0.tgz#5f0310e18b8be898cc07009295a30ae41e91e6f5" + integrity sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA== + dependencies: + pify "^4.0.1" + semver "^5.6.0" + +make-dir@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" + integrity sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw== + dependencies: + semver "^6.0.0" + +make-error@1.x, make-error@^1.1.1: + version "1.3.6" + resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" + integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw== + +make-fetch-happen@^8.0.9: + version "8.0.14" + resolved "https://registry.yarnpkg.com/make-fetch-happen/-/make-fetch-happen-8.0.14.tgz#aaba73ae0ab5586ad8eaa68bd83332669393e222" + integrity sha512-EsS89h6l4vbfJEtBZnENTOFk8mCRpY5ru36Xe5bcX1KYIli2mkSHqoFsp5O1wMDvTJJzxe/4THpCTtygjeeGWQ== + dependencies: + agentkeepalive "^4.1.3" + cacache "^15.0.5" + http-cache-semantics "^4.1.0" + http-proxy-agent "^4.0.1" + https-proxy-agent "^5.0.0" + is-lambda "^1.0.1" + lru-cache "^6.0.0" + minipass "^3.1.3" + minipass-collect "^1.0.2" + minipass-fetch "^1.3.2" + minipass-flush "^1.0.5" + minipass-pipeline "^1.2.4" + promise-retry "^2.0.1" + socks-proxy-agent "^5.0.0" + ssri "^8.0.0" + +make-fetch-happen@^9.0.1: + version "9.0.4" + resolved "https://registry.yarnpkg.com/make-fetch-happen/-/make-fetch-happen-9.0.4.tgz#ceaa100e60e0ef9e8d1ede94614bb2ba83c8bb24" + integrity sha512-sQWNKMYqSmbAGXqJg2jZ+PmHh5JAybvwu0xM8mZR/bsTjGiTASj3ldXJV7KFHy1k/IJIBkjxQFoWIVsv9+PQMg== + dependencies: + agentkeepalive "^4.1.3" + cacache "^15.2.0" + http-cache-semantics "^4.1.0" + http-proxy-agent "^4.0.1" + https-proxy-agent "^5.0.0" + is-lambda "^1.0.1" + lru-cache "^6.0.0" + minipass "^3.1.3" + minipass-collect "^1.0.2" + minipass-fetch "^1.3.2" + minipass-flush "^1.0.5" + minipass-pipeline "^1.2.4" + negotiator "^0.6.2" + promise-retry "^2.0.1" + socks-proxy-agent "^5.0.0" + ssri "^8.0.0" + +make-iterator@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/make-iterator/-/make-iterator-1.0.1.tgz#29b33f312aa8f547c4a5e490f56afcec99133ad6" + integrity sha512-pxiuXh0iVEq7VM7KMIhs5gxsfxCux2URptUQaXo4iZZJxBAzTPOLE2BumO5dbfVYq/hBJFBR/a1mFDmOx5AGmw== + dependencies: + kind-of "^6.0.2" + +makeerror@1.0.x: + version "1.0.11" + resolved "https://registry.yarnpkg.com/makeerror/-/makeerror-1.0.11.tgz#e01a5c9109f2af79660e4e8b9587790184f5a96c" + integrity sha1-4BpckQnyr3lmDk6LlYd5AYT1qWw= + dependencies: + tmpl "1.0.x" + +map-cache@^0.2.0, map-cache@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/map-cache/-/map-cache-0.2.2.tgz#c32abd0bd6525d9b051645bb4f26ac5dc98a0dbf" + integrity sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8= + +map-obj@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d" + integrity sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0= + +map-obj@^4.0.0, map-obj@^4.1.0: + version "4.2.1" + resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-4.2.1.tgz#e4ea399dbc979ae735c83c863dd31bdf364277b7" + integrity sha512-+WA2/1sPmDj1dlvvJmB5G6JKfY9dpn7EVBUL06+y6PoljPkh+6V1QihwxNkbcGxCRjt2b0F9K0taiCuo7MbdFQ== + +map-visit@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/map-visit/-/map-visit-1.0.0.tgz#ecdca8f13144e660f1b5bd41f12f3479d98dfb8f" + integrity sha1-7Nyo8TFE5mDxtb1B8S80edmN+48= + dependencies: + object-visit "^1.0.0" + +marked@^2.1.1: + version "2.1.3" + resolved "https://registry.yarnpkg.com/marked/-/marked-2.1.3.tgz#bd017cef6431724fd4b27e0657f5ceb14bff3753" + integrity sha512-/Q+7MGzaETqifOMWYEA7HVMaZb4XbcRfaOzcSsHZEith83KGlvaSG33u0SKu89Mj5h+T8V2hM+8O45Qc5XTgwA== + +matchdep@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/matchdep/-/matchdep-2.0.0.tgz#c6f34834a0d8dbc3b37c27ee8bbcb27c7775582e" + integrity sha1-xvNINKDY28OzfCfui7yyfHd1WC4= + dependencies: + findup-sync "^2.0.0" + micromatch "^3.0.4" + resolve "^1.4.0" + stack-trace "0.0.10" + +math-random@^1.0.1: + version "1.0.4" + resolved "https://registry.yarnpkg.com/math-random/-/math-random-1.0.4.tgz#5dd6943c938548267016d4e34f057583080c514c" + integrity sha512-rUxjysqif/BZQH2yhd5Aaq7vXMSx9NdEsQcyA07uEzIvxgI7zIr33gGsh+RU0/XjmQpCW7RsVof1vlkvQVCK5A== + +memfs@3.2.2: + version "3.2.2" + resolved "https://registry.yarnpkg.com/memfs/-/memfs-3.2.2.tgz#5de461389d596e3f23d48bb7c2afb6161f4df40e" + integrity sha512-RE0CwmIM3CEvpcdK3rZ19BC4E6hv9kADkMN5rPduRak58cNArWLi/9jFLsa4rhsjfVxMP3v0jO7FHXq7SvFY5Q== + dependencies: + fs-monkey "1.0.3" + +memoizee@0.4.X: + version "0.4.15" + resolved "https://registry.yarnpkg.com/memoizee/-/memoizee-0.4.15.tgz#e6f3d2da863f318d02225391829a6c5956555b72" + integrity sha512-UBWmJpLZd5STPm7PMUlOw/TSy972M+z8gcyQ5veOnSDRREz/0bmpyTfKt3/51DhEBqCZQn1udM/5flcSPYhkdQ== + dependencies: + d "^1.0.1" + es5-ext "^0.10.53" + es6-weak-map "^2.0.3" + event-emitter "^0.3.5" + is-promise "^2.2.2" + lru-queue "^0.1.0" + next-tick "^1.1.0" + timers-ext "^0.1.7" + +memorystream@^0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/memorystream/-/memorystream-0.3.1.tgz#86d7090b30ce455d63fbae12dda51a47ddcaf9b2" + integrity sha1-htcJCzDORV1j+64S3aUaR93K+bI= + +meow@^10.1.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/meow/-/meow-10.1.0.tgz#43edce35b3c5b7056d74bd9d63897220d3c190a6" + integrity sha512-bks/XR5OSTWcPZbJ/NsE2uCWQJ/ejqv8M9XOYxzhufBjreUMuz7S5ApDN5knzQce/4sLT5QoOQc6BbD5O0yP/w== + dependencies: + "@types/minimist" "^1.2.2" + camelcase-keys "^7.0.0" + decamelize "^5.0.0" + decamelize-keys "^1.1.0" + hard-rejection "^2.1.0" + minimist-options "4.1.0" + normalize-package-data "^3.0.2" + read-pkg-up "^8.0.0" + redent "^4.0.0" + trim-newlines "^4.0.2" + type-fest "^1.2.2" + yargs-parser "^20.2.9" + +meow@^7.0.0: + version "7.1.1" + resolved "https://registry.yarnpkg.com/meow/-/meow-7.1.1.tgz#7c01595e3d337fcb0ec4e8eed1666ea95903d306" + integrity sha512-GWHvA5QOcS412WCo8vwKDlTelGLsCGBVevQB5Kva961rmNfun0PCbv5+xta2kUMFJyR8/oWnn7ddeKdosbAPbA== + dependencies: + "@types/minimist" "^1.2.0" + camelcase-keys "^6.2.2" + decamelize-keys "^1.1.0" + hard-rejection "^2.1.0" + minimist-options "4.1.0" + normalize-package-data "^2.5.0" + read-pkg-up "^7.0.1" + redent "^3.0.0" + trim-newlines "^3.0.0" + type-fest "^0.13.1" + yargs-parser "^18.1.3" + +meow@^8.0.0: + version "8.1.2" + resolved "https://registry.yarnpkg.com/meow/-/meow-8.1.2.tgz#bcbe45bda0ee1729d350c03cffc8395a36c4e897" + integrity sha512-r85E3NdZ+mpYk1C6RjPFEMSE+s1iZMuHtsHAqY0DT3jZczl0diWUZ8g6oU7h0M9cD2EL+PzaYghhCLzR0ZNn5Q== + dependencies: + "@types/minimist" "^1.2.0" + camelcase-keys "^6.2.2" + decamelize-keys "^1.1.0" + hard-rejection "^2.1.0" + minimist-options "4.1.0" + normalize-package-data "^3.0.0" + read-pkg-up "^7.0.1" + redent "^3.0.0" + trim-newlines "^3.0.0" + type-fest "^0.18.0" + yargs-parser "^20.2.3" + +merge-stream@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60" + integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w== + +merge2@^1.2.3, merge2@^1.3.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.4.1.tgz#4368892f885e907455a6fd7dc55c0c9d404990ae" + integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== + +micromatch@^3.0.4, micromatch@^3.1.10, micromatch@^3.1.4: + version "3.1.10" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-3.1.10.tgz#70859bc95c9840952f359a068a3fc49f9ecfac23" + integrity sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg== + dependencies: + arr-diff "^4.0.0" + array-unique "^0.3.2" + braces "^2.3.1" + define-property "^2.0.2" + extend-shallow "^3.0.2" + extglob "^2.0.4" + fragment-cache "^0.2.1" + kind-of "^6.0.2" + nanomatch "^1.2.9" + object.pick "^1.3.0" + regex-not "^1.0.0" + snapdragon "^0.8.1" + to-regex "^3.0.2" + +micromatch@^4.0.2, micromatch@^4.0.4: + version "4.0.4" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.4.tgz#896d519dfe9db25fce94ceb7a500919bf881ebf9" + integrity sha512-pRmzw/XUcwXGpD9aI9q/0XOwLNygjETJ8y0ao0wdqprrzDa4YnxLcz7fQRZr8voh8V10kGhABbNcHVk5wHgWwg== + dependencies: + braces "^3.0.1" + picomatch "^2.2.3" + +mime-db@1.48.0: + version "1.48.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.48.0.tgz#e35b31045dd7eada3aaad537ed88a33afbef2d1d" + integrity sha512-FM3QwxV+TnZYQ2aRqhlKBMHxk10lTbMt3bBkMAp54ddrNeVSfcQYOOKuGuy3Ddrm38I04If834fOUSq1yzslJQ== + +mime-types@^2.1.12, mime-types@~2.1.19: + version "2.1.31" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.31.tgz#a00d76b74317c61f9c2db2218b8e9f8e9c5c9e6b" + integrity sha512-XGZnNzm3QvgKxa8dpzyhFTHmpP3l5YNusmne07VUOXxou9CqUqYa/HBy124RqtVh/O2pECas/MOcsDgpilPOPg== + dependencies: + mime-db "1.48.0" + +mimic-fn@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" + integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== + +min-indent@^1.0.0, min-indent@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/min-indent/-/min-indent-1.0.1.tgz#a63f681673b30571fbe8bc25686ae746eefa9869" + integrity sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg== + +minimatch@^3.0.0, minimatch@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" + integrity sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA== + dependencies: + brace-expansion "^1.1.7" + +minimist-options@4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/minimist-options/-/minimist-options-4.1.0.tgz#c0655713c53a8a2ebd77ffa247d342c40f010619" + integrity sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A== + dependencies: + arrify "^1.0.1" + is-plain-obj "^1.1.0" + kind-of "^6.0.3" + +minimist@1.x, minimist@^1.2.0, minimist@^1.2.5: + version "1.2.5" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" + integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== + +minipass-collect@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/minipass-collect/-/minipass-collect-1.0.2.tgz#22b813bf745dc6edba2576b940022ad6edc8c617" + integrity sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA== + dependencies: + minipass "^3.0.0" + +minipass-fetch@^1.3.0, minipass-fetch@^1.3.2: + version "1.3.4" + resolved "https://registry.yarnpkg.com/minipass-fetch/-/minipass-fetch-1.3.4.tgz#63f5af868a38746ca7b33b03393ddf8c291244fe" + integrity sha512-TielGogIzbUEtd1LsjZFs47RWuHHfhl6TiCx1InVxApBAmQ8bL0dL5ilkLGcRvuyW/A9nE+Lvn855Ewz8S0PnQ== + dependencies: + minipass "^3.1.0" + minipass-sized "^1.0.3" + minizlib "^2.0.0" + optionalDependencies: + encoding "^0.1.12" + +minipass-flush@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/minipass-flush/-/minipass-flush-1.0.5.tgz#82e7135d7e89a50ffe64610a787953c4c4cbb373" + integrity sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw== + dependencies: + minipass "^3.0.0" + +minipass-json-stream@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/minipass-json-stream/-/minipass-json-stream-1.0.1.tgz#7edbb92588fbfc2ff1db2fc10397acb7b6b44aa7" + integrity sha512-ODqY18UZt/I8k+b7rl2AENgbWE8IDYam+undIJONvigAz8KR5GWblsFTEfQs0WODsjbSXWlm+JHEv8Gr6Tfdbg== + dependencies: + jsonparse "^1.3.1" + minipass "^3.0.0" + +minipass-pipeline@^1.2.2, minipass-pipeline@^1.2.4: + version "1.2.4" + resolved "https://registry.yarnpkg.com/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz#68472f79711c084657c067c5c6ad93cddea8214c" + integrity sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A== + dependencies: + minipass "^3.0.0" + +minipass-sized@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/minipass-sized/-/minipass-sized-1.0.3.tgz#70ee5a7c5052070afacfbc22977ea79def353b70" + integrity sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g== + dependencies: + minipass "^3.0.0" + +minipass@^2.6.0, minipass@^2.8.6, minipass@^2.9.0: + version "2.9.0" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-2.9.0.tgz#e713762e7d3e32fed803115cf93e04bca9fcc9a6" + integrity sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg== + dependencies: + safe-buffer "^5.1.2" + yallist "^3.0.0" + +minipass@^3.0.0, minipass@^3.1.0, minipass@^3.1.1, minipass@^3.1.3: + version "3.1.3" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.1.3.tgz#7d42ff1f39635482e15f9cdb53184deebd5815fd" + integrity sha512-Mgd2GdMVzY+x3IJ+oHnVM+KG3lA5c8tnabyJKmHSaG2kAGpudxuOf8ToDkhumF7UzME7DecbQE9uOZhNm7PuJg== + dependencies: + yallist "^4.0.0" + +minizlib@^1.2.1: + version "1.3.3" + resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-1.3.3.tgz#2290de96818a34c29551c8a8d301216bd65a861d" + integrity sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q== + dependencies: + minipass "^2.9.0" + +minizlib@^2.0.0, minizlib@^2.1.1: + version "2.1.2" + resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-2.1.2.tgz#e90d3466ba209b932451508a11ce3d3632145931" + integrity sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg== + dependencies: + minipass "^3.0.0" + yallist "^4.0.0" + +mixin-deep@^1.2.0: + version "1.3.2" + resolved "https://registry.yarnpkg.com/mixin-deep/-/mixin-deep-1.3.2.tgz#1120b43dc359a785dce65b55b82e257ccf479566" + integrity sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA== + dependencies: + for-in "^1.0.2" + is-extendable "^1.0.1" + +mkdirp-infer-owner@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/mkdirp-infer-owner/-/mkdirp-infer-owner-2.0.0.tgz#55d3b368e7d89065c38f32fd38e638f0ab61d316" + integrity sha512-sdqtiFt3lkOaYvTXSRIUjkIdPTcxgv5+fgqYE/5qgwdw12cOrAuzzgzvVExIkH/ul1oeHN3bCLOWSG3XOqbKKw== + dependencies: + chownr "^2.0.0" + infer-owner "^1.0.4" + mkdirp "^1.0.3" + +mkdirp@1.0.4, mkdirp@1.x, mkdirp@^1.0.3, mkdirp@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" + integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== + +mkdirp@^0.5.0, mkdirp@^0.5.1: + version "0.5.5" + resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" + integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ== + dependencies: + minimist "^1.2.5" + +modify-values@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/modify-values/-/modify-values-1.0.1.tgz#b3939fa605546474e3e3e3c63d64bd43b4ee6022" + integrity sha512-xV2bxeN6F7oYjZWTe/YPAy6MN2M+sL4u/Rlm2AHCIVGfo2p1yGmBHQ6vHehl4bRTZBdHu3TSkWdYgkwpYzAGSw== + +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= + +ms@2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" + integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== + +ms@^2.0.0, ms@^2.1.1: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + +multimatch@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/multimatch/-/multimatch-5.0.0.tgz#932b800963cea7a31a033328fa1e0c3a1874dbe6" + integrity sha512-ypMKuglUrZUD99Tk2bUQ+xNQj43lPEfAeX2o9cTteAmShXy2VHDJpuwu1o0xqoKCt9jLVAvwyFKdLTPXKAfJyA== + dependencies: + "@types/minimatch" "^3.0.3" + array-differ "^3.0.0" + array-union "^2.1.0" + arrify "^2.0.1" + minimatch "^3.0.4" + +multistream@4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/multistream/-/multistream-4.1.0.tgz#7bf00dfd119556fbc153cff3de4c6d477909f5a8" + integrity sha512-J1XDiAmmNpRCBfIWJv+n0ymC4ABcf/Pl+5YvC5B/D2f/2+8PtHvCNxMPKiQcZyi922Hq69J2YOpb1pTywfifyw== + dependencies: + once "^1.4.0" + readable-stream "^3.6.0" + +mute-stdout@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/mute-stdout/-/mute-stdout-1.0.1.tgz#acb0300eb4de23a7ddeec014e3e96044b3472331" + integrity sha512-kDcwXR4PS7caBpuRYYBUz9iVixUk3anO3f5OYFiIPwK/20vCzKCHyKoulbiDY1S53zD2bxUpxN/IJ+TnXjfvxg== + +mute-stream@0.0.8, mute-stream@~0.0.4: + version "0.0.8" + resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d" + integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA== + +nan@^2.12.1: + version "2.14.2" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.2.tgz#f5376400695168f4cc694ac9393d0c9585eeea19" + integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ== + +nanomatch@^1.2.9: + version "1.2.13" + resolved "https://registry.yarnpkg.com/nanomatch/-/nanomatch-1.2.13.tgz#b87a8aa4fc0de8fe6be88895b38983ff265bd119" + integrity sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA== + dependencies: + arr-diff "^4.0.0" + array-unique "^0.3.2" + define-property "^2.0.2" + extend-shallow "^3.0.2" + fragment-cache "^0.2.1" + is-windows "^1.0.2" + kind-of "^6.0.2" + object.pick "^1.3.0" + regex-not "^1.0.0" + snapdragon "^0.8.1" + to-regex "^3.0.1" + +natural-compare@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" + integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= + +negotiator@^0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb" + integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw== + +neo-async@^2.6.0: + version "2.6.2" + resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" + integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== + +nested-error-stacks@^2.0.0, nested-error-stacks@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/nested-error-stacks/-/nested-error-stacks-2.1.0.tgz#0fbdcf3e13fe4994781280524f8b96b0cdff9c61" + integrity sha512-AO81vsIO1k1sM4Zrd6Hu7regmJN1NSiAja10gc4bX3F0wd+9rQmcuHQaHVQCYIEC8iFXnE+mavh23GOt7wBgug== + +next-tick@1, next-tick@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.1.0.tgz#1836ee30ad56d67ef281b22bd199f709449b35eb" + integrity sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ== + +next-tick@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.0.0.tgz#ca86d1fe8828169b0120208e3dc8424b9db8342c" + integrity sha1-yobR/ogoFpsBICCOPchCS524NCw= + +nice-try@^1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" + integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== + +node-fetch@^2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052" + integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== + +node-gyp@^5.0.2: + version "5.1.1" + resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-5.1.1.tgz#eb915f7b631c937d282e33aed44cb7a025f62a3e" + integrity sha512-WH0WKGi+a4i4DUt2mHnvocex/xPLp9pYt5R6M2JdFB7pJ7Z34hveZ4nDTGTiLXCkitA9T8HFZjhinBCiVHYcWw== + dependencies: + env-paths "^2.2.0" + glob "^7.1.4" + graceful-fs "^4.2.2" + mkdirp "^0.5.1" + nopt "^4.0.1" + npmlog "^4.1.2" + request "^2.88.0" + rimraf "^2.6.3" + semver "^5.7.1" + tar "^4.4.12" + which "^1.3.1" + +node-gyp@^7.1.0: + version "7.1.2" + resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-7.1.2.tgz#21a810aebb187120251c3bcec979af1587b188ae" + integrity sha512-CbpcIo7C3eMu3dL1c3d0xw449fHIGALIJsRP4DDPHpyiW8vcriNY7ubh9TE4zEKfSxscY7PjeFnshE7h75ynjQ== + dependencies: + env-paths "^2.2.0" + glob "^7.1.4" + graceful-fs "^4.2.3" + nopt "^5.0.0" + npmlog "^4.1.2" + request "^2.88.2" + rimraf "^3.0.2" + semver "^7.3.2" + tar "^6.0.2" + which "^2.0.2" + +node-int64@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b" + integrity sha1-h6kGXNs1XTGC2PlM4RGIuCXGijs= + +node-modules-regexp@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/node-modules-regexp/-/node-modules-regexp-1.0.0.tgz#8d9dbe28964a4ac5712e9131642107c71e90ec40" + integrity sha1-jZ2+KJZKSsVxLpExZCEHxx6Q7EA= + +node-releases@^1.1.71: + version "1.1.73" + resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-1.1.73.tgz#dd4e81ddd5277ff846b80b52bb40c49edf7a7b20" + integrity sha512-uW7fodD6pyW2FZNZnp/Z3hvWKeEW1Y8R1+1CnErE8cXFXzl5blBOoVB41CvMer6P6Q0S5FXDwcHgFd1Wj0U9zg== + +nopt@^4.0.1: + version "4.0.3" + resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.3.tgz#a375cad9d02fd921278d954c2254d5aa57e15e48" + integrity sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg== + dependencies: + abbrev "1" + osenv "^0.1.4" + +nopt@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/nopt/-/nopt-5.0.0.tgz#530942bb58a512fccafe53fe210f13a25355dc88" + integrity sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ== + dependencies: + abbrev "1" + +normalize-package-data@^2.0.0, normalize-package-data@^2.3.2, normalize-package-data@^2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8" + integrity sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA== + dependencies: + hosted-git-info "^2.1.4" + resolve "^1.10.0" + semver "2 || 3 || 4 || 5" + validate-npm-package-license "^3.0.1" + +normalize-package-data@^3.0.0, normalize-package-data@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-3.0.2.tgz#cae5c410ae2434f9a6c1baa65d5bc3b9366c8699" + integrity sha512-6CdZocmfGaKnIHPVFhJJZ3GuR8SsLKvDANFp47Jmy51aKIr8akjAWTSxtpI+MBgBFdSMRyo4hMpDlT6dTffgZg== + dependencies: + hosted-git-info "^4.0.1" + resolve "^1.20.0" + semver "^7.3.4" + validate-npm-package-license "^3.0.1" + +normalize-path@^2.0.1, normalize-path@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-2.1.1.tgz#1ab28b556e198363a8c1a6f7e6fa20137fe6aed9" + integrity sha1-GrKLVW4Zg2Oowab35vogE3/mrtk= + dependencies: + remove-trailing-separator "^1.0.1" + +normalize-path@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" + integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA== + +normalize-url@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-6.1.0.tgz#40d0885b535deffe3f3147bec877d05fe4c5668a" + integrity sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A== + +now-and-later@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/now-and-later/-/now-and-later-2.0.1.tgz#8e579c8685764a7cc02cb680380e94f43ccb1f7c" + integrity sha512-KGvQ0cB70AQfg107Xvs/Fbu+dGmZoTRJp2TaPwcwQm3/7PteUyN2BCgk8KBMPGBUXZdVwyWS8fDCGFygBm19UQ== + dependencies: + once "^1.3.2" + +npm-bundled@^1.1.1: + version "1.1.2" + resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.1.2.tgz#944c78789bd739035b70baa2ca5cc32b8d860bc1" + integrity sha512-x5DHup0SuyQcmL3s7Rx/YQ8sbw/Hzg0rj48eN0dV7hf5cmQq5PXIeioroH3raV1QC1yh3uTYuMThvEQF3iKgGQ== + dependencies: + npm-normalize-package-bin "^1.0.1" + +npm-install-checks@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/npm-install-checks/-/npm-install-checks-4.0.0.tgz#a37facc763a2fde0497ef2c6d0ac7c3fbe00d7b4" + integrity sha512-09OmyDkNLYwqKPOnbI8exiOZU2GVVmQp7tgez2BPi5OZC8M82elDAps7sxC4l//uSUtotWqoEIDwjRvWH4qz8w== + dependencies: + semver "^7.1.1" + +npm-lifecycle@^3.1.5: + version "3.1.5" + resolved "https://registry.yarnpkg.com/npm-lifecycle/-/npm-lifecycle-3.1.5.tgz#9882d3642b8c82c815782a12e6a1bfeed0026309" + integrity sha512-lDLVkjfZmvmfvpvBzA4vzee9cn+Me4orq0QF8glbswJVEbIcSNWib7qGOffolysc3teCqbbPZZkzbr3GQZTL1g== + dependencies: + byline "^5.0.0" + graceful-fs "^4.1.15" + node-gyp "^5.0.2" + resolve-from "^4.0.0" + slide "^1.1.6" + uid-number "0.0.6" + umask "^1.1.0" + which "^1.3.1" + +npm-normalize-package-bin@^1.0.0, npm-normalize-package-bin@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/npm-normalize-package-bin/-/npm-normalize-package-bin-1.0.1.tgz#6e79a41f23fd235c0623218228da7d9c23b8f6e2" + integrity sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA== + +npm-package-arg@^8.0.0, npm-package-arg@^8.0.1, npm-package-arg@^8.1.0, npm-package-arg@^8.1.2: + version "8.1.5" + resolved "https://registry.yarnpkg.com/npm-package-arg/-/npm-package-arg-8.1.5.tgz#3369b2d5fe8fdc674baa7f1786514ddc15466e44" + integrity sha512-LhgZrg0n0VgvzVdSm1oiZworPbTxYHUJCgtsJW8mGvlDpxTM1vSJc3m5QZeUkhAHIzbz3VCHd/R4osi1L1Tg/Q== + dependencies: + hosted-git-info "^4.0.1" + semver "^7.3.4" + validate-npm-package-name "^3.0.0" + +npm-packlist@^2.1.4: + version "2.2.2" + resolved "https://registry.yarnpkg.com/npm-packlist/-/npm-packlist-2.2.2.tgz#076b97293fa620f632833186a7a8f65aaa6148c8" + integrity sha512-Jt01acDvJRhJGthnUJVF/w6gumWOZxO7IkpY/lsX9//zqQgnF7OJaxgQXcerd4uQOLu7W5bkb4mChL9mdfm+Zg== + dependencies: + glob "^7.1.6" + ignore-walk "^3.0.3" + npm-bundled "^1.1.1" + npm-normalize-package-bin "^1.0.1" + +npm-pick-manifest@^6.0.0, npm-pick-manifest@^6.1.1: + version "6.1.1" + resolved "https://registry.yarnpkg.com/npm-pick-manifest/-/npm-pick-manifest-6.1.1.tgz#7b5484ca2c908565f43b7f27644f36bb816f5148" + integrity sha512-dBsdBtORT84S8V8UTad1WlUyKIY9iMsAmqxHbLdeEeBNMLQDlDWWra3wYUx9EBEIiG/YwAy0XyNHDd2goAsfuA== + dependencies: + npm-install-checks "^4.0.0" + npm-normalize-package-bin "^1.0.1" + npm-package-arg "^8.1.2" + semver "^7.3.4" + +npm-registry-fetch@^11.0.0: + version "11.0.0" + resolved "https://registry.yarnpkg.com/npm-registry-fetch/-/npm-registry-fetch-11.0.0.tgz#68c1bb810c46542760d62a6a965f85a702d43a76" + integrity sha512-jmlgSxoDNuhAtxUIG6pVwwtz840i994dL14FoNVZisrmZW5kWd63IUTNv1m/hyRSGSqWjCUp/YZlS1BJyNp9XA== + dependencies: + make-fetch-happen "^9.0.1" + minipass "^3.1.3" + minipass-fetch "^1.3.0" + minipass-json-stream "^1.0.1" + minizlib "^2.0.0" + npm-package-arg "^8.0.0" + +npm-registry-fetch@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/npm-registry-fetch/-/npm-registry-fetch-9.0.0.tgz#86f3feb4ce00313bc0b8f1f8f69daae6face1661" + integrity sha512-PuFYYtnQ8IyVl6ib9d3PepeehcUeHN9IO5N/iCRhyg9tStQcqGQBRVHmfmMWPDERU3KwZoHFvbJ4FPXPspvzbA== + dependencies: + "@npmcli/ci-detect" "^1.0.0" + lru-cache "^6.0.0" + make-fetch-happen "^8.0.9" + minipass "^3.1.3" + minipass-fetch "^1.3.0" + minipass-json-stream "^1.0.1" + minizlib "^2.0.0" + npm-package-arg "^8.0.0" + +npm-run-all@4.1.5: + version "4.1.5" + resolved "https://registry.yarnpkg.com/npm-run-all/-/npm-run-all-4.1.5.tgz#04476202a15ee0e2e214080861bff12a51d98fba" + integrity sha512-Oo82gJDAVcaMdi3nuoKFavkIHBRVqQ1qvMb+9LHk/cF4P6B2m8aP04hGf7oL6wZ9BuGwX1onlLhpuoofSyoQDQ== + dependencies: + ansi-styles "^3.2.1" + chalk "^2.4.1" + cross-spawn "^6.0.5" + memorystream "^0.3.1" + minimatch "^3.0.4" + pidtree "^0.3.0" + read-pkg "^3.0.0" + shell-quote "^1.6.1" + string.prototype.padend "^3.0.0" + +npm-run-path@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-4.0.1.tgz#b7ecd1e5ed53da8e37a55e1c2269e0b97ed748ea" + integrity sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw== + dependencies: + path-key "^3.0.0" + +npmlog@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" + integrity sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg== + dependencies: + are-we-there-yet "~1.1.2" + console-control-strings "~1.1.0" + gauge "~2.7.3" + set-blocking "~2.0.0" + +number-is-nan@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" + integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0= + +nwsapi@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.0.tgz#204879a9e3d068ff2a55139c2c772780681a38b7" + integrity sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ== + +oauth-sign@~0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" + integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== + +object-assign@4.X, object-assign@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" + integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM= + +object-copy@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/object-copy/-/object-copy-0.1.0.tgz#7e7d858b781bd7c991a41ba975ed3812754e998c" + integrity sha1-fn2Fi3gb18mRpBupde04EnVOmYw= + dependencies: + copy-descriptor "^0.1.0" + define-property "^0.2.5" + kind-of "^3.0.3" + +object-inspect@^1.10.3, object-inspect@^1.9.0: + version "1.11.0" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.11.0.tgz#9dceb146cedd4148a0d9e51ab88d34cf509922b1" + integrity sha512-jp7ikS6Sd3GxQfZJPyH3cjcbJF6GZPClgdV+EFygjFLQ5FmW/dRUnTd9PQ9k0JhoNDabWFbpF1yCdSWCC6gexg== + +object-keys@^1.0.12, object-keys@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" + integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA== + +object-visit@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/object-visit/-/object-visit-1.0.1.tgz#f79c4493af0c5377b59fe39d395e41042dd045bb" + integrity sha1-95xEk68MU3e1n+OdOV5BBC3QRbs= + dependencies: + isobject "^3.0.0" + +object.assign@^4.0.4, object.assign@^4.1.0, object.assign@^4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.2.tgz#0ed54a342eceb37b38ff76eb831a0e788cb63940" + integrity sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ== + dependencies: + call-bind "^1.0.0" + define-properties "^1.1.3" + has-symbols "^1.0.1" + object-keys "^1.1.1" + +object.defaults@^1.0.0, object.defaults@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/object.defaults/-/object.defaults-1.1.0.tgz#3a7f868334b407dea06da16d88d5cd29e435fecf" + integrity sha1-On+GgzS0B96gbaFtiNXNKeQ1/s8= + dependencies: + array-each "^1.0.1" + array-slice "^1.0.0" + for-own "^1.0.0" + isobject "^3.0.0" + +object.getownpropertydescriptors@^2.0.3: + version "2.1.2" + resolved "https://registry.yarnpkg.com/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.2.tgz#1bd63aeacf0d5d2d2f31b5e393b03a7c601a23f7" + integrity sha512-WtxeKSzfBjlzL+F9b7M7hewDzMwy+C8NRssHd1YrNlzHzIDrXcXiNOMrezdAEM4UXixgV+vvnyBeN7Rygl2ttQ== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.3" + es-abstract "^1.18.0-next.2" + +object.map@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/object.map/-/object.map-1.0.1.tgz#cf83e59dc8fcc0ad5f4250e1f78b3b81bd801d37" + integrity sha1-z4Plncj8wK1fQlDh94s7gb2AHTc= + dependencies: + for-own "^1.0.0" + make-iterator "^1.0.0" + +object.pick@^1.2.0, object.pick@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/object.pick/-/object.pick-1.3.0.tgz#87a10ac4c1694bd2e1cbf53591a66141fb5dd747" + integrity sha1-h6EKxMFpS9Lhy/U1kaZhQftd10c= + dependencies: + isobject "^3.0.1" + +object.reduce@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/object.reduce/-/object.reduce-1.0.1.tgz#6fe348f2ac7fa0f95ca621226599096825bb03ad" + integrity sha1-b+NI8qx/oPlcpiEiZZkJaCW7A60= + dependencies: + for-own "^1.0.0" + make-iterator "^1.0.0" + +once@^1.3.0, once@^1.3.1, once@^1.3.2, once@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" + integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= + dependencies: + wrappy "1" + +onetime@^5.1.0, onetime@^5.1.2: + version "5.1.2" + resolved "https://registry.yarnpkg.com/onetime/-/onetime-5.1.2.tgz#d0e96ebb56b07476df1dd9c4806e5237985ca45e" + integrity sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg== + dependencies: + mimic-fn "^2.1.0" + +onigasm@^2.2.5: + version "2.2.5" + resolved "https://registry.yarnpkg.com/onigasm/-/onigasm-2.2.5.tgz#cc4d2a79a0fa0b64caec1f4c7ea367585a676892" + integrity sha512-F+th54mPc0l1lp1ZcFMyL/jTs2Tlq4SqIHKIXGZOR/VkHkF9A7Fr5rRr5+ZG/lWeRsyrClLYRq7s/yFQ/XhWCA== + dependencies: + lru-cache "^5.1.1" + +optionator@^0.8.1: + version "0.8.3" + resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.3.tgz#84fa1d036fe9d3c7e21d99884b601167ec8fb495" + integrity sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA== + dependencies: + deep-is "~0.1.3" + fast-levenshtein "~2.0.6" + levn "~0.3.0" + prelude-ls "~1.1.2" + type-check "~0.3.2" + word-wrap "~1.2.3" + +optionator@^0.9.1: + version "0.9.1" + resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.1.tgz#4f236a6373dae0566a6d43e1326674f50c291499" + integrity sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw== + dependencies: + deep-is "^0.1.3" + fast-levenshtein "^2.0.6" + levn "^0.4.1" + prelude-ls "^1.2.1" + type-check "^0.4.0" + word-wrap "^1.2.3" + +ordered-read-streams@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/ordered-read-streams/-/ordered-read-streams-1.0.1.tgz#77c0cb37c41525d64166d990ffad7ec6a0e1363e" + integrity sha1-d8DLN8QVJdZBZtmQ/61+xqDhNj4= + dependencies: + readable-stream "^2.0.1" + +os-homedir@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" + integrity sha1-/7xJiDNuDoM94MFox+8VISGqf7M= + +os-locale@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/os-locale/-/os-locale-1.4.0.tgz#20f9f17ae29ed345e8bde583b13d2009803c14d9" + integrity sha1-IPnxeuKe00XoveWDsT0gCYA8FNk= + dependencies: + lcid "^1.0.0" + +os-tmpdir@^1.0.0, os-tmpdir@~1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" + integrity sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ= + +osenv@^0.1.4: + version "0.1.5" + resolved "https://registry.yarnpkg.com/osenv/-/osenv-0.1.5.tgz#85cdfafaeb28e8677f416e287592b5f3f49ea410" + integrity sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g== + dependencies: + os-homedir "^1.0.0" + os-tmpdir "^1.0.0" + +p-all@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/p-all/-/p-all-2.1.0.tgz#91419be56b7dee8fe4c5db875d55e0da084244a0" + integrity sha512-HbZxz5FONzz/z2gJfk6bFca0BCiSRF8jU3yCsWOen/vR6lZjfPOu/e7L3uFzTW1i0H8TlC3vqQstEJPQL4/uLA== + dependencies: + p-map "^2.0.0" + +p-each-series@^2.1.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/p-each-series/-/p-each-series-2.2.0.tgz#105ab0357ce72b202a8a8b94933672657b5e2a9a" + integrity sha512-ycIL2+1V32th+8scbpTvyHNaHe02z0sjgh91XXjAk+ZeXoPN4Z46DVUnzdso0aX4KckKw0FNNFHdjZ2UsZvxiA== + +p-event@^4.1.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/p-event/-/p-event-4.2.0.tgz#af4b049c8acd91ae81083ebd1e6f5cae2044c1b5" + integrity sha512-KXatOjCRXXkSePPb1Nbi0p0m+gQAwdlbhi4wQKJPI1HsMQS9g+Sqp2o+QHziPr7eYJyOZet836KoHEVM1mwOrQ== + dependencies: + p-timeout "^3.1.0" + +p-filter@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/p-filter/-/p-filter-2.1.0.tgz#1b1472562ae7a0f742f0f3d3d3718ea66ff9c09c" + integrity sha512-ZBxxZ5sL2HghephhpGAQdoskxplTwr7ICaehZwLIlfL6acuVgZPm8yBNuRAFBGEqtD/hmUeq9eqLg2ys9Xr/yw== + dependencies: + p-map "^2.0.0" + +p-finally@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" + integrity sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4= + +p-limit@^1.1.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-1.3.0.tgz#b86bd5f0c25690911c7590fcbfc2010d54b3ccb8" + integrity sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q== + dependencies: + p-try "^1.0.0" + +p-limit@^2.2.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1" + integrity sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w== + dependencies: + p-try "^2.0.0" + +p-limit@^3.0.2: + version "3.1.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" + integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== + dependencies: + yocto-queue "^0.1.0" + +p-locate@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43" + integrity sha1-IKAQOyIqcMj9OcwuWAaA893l7EM= + dependencies: + p-limit "^1.1.0" + +p-locate@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-4.1.0.tgz#a3428bb7088b3a60292f66919278b7c297ad4f07" + integrity sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A== + dependencies: + p-limit "^2.2.0" + +p-locate@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834" + integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw== + dependencies: + p-limit "^3.0.2" + +p-map-series@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/p-map-series/-/p-map-series-2.1.0.tgz#7560d4c452d9da0c07e692fdbfe6e2c81a2a91f2" + integrity sha512-RpYIIK1zXSNEOdwxcfe7FdvGcs7+y5n8rifMhMNWvaxRNMPINJHF5GDeuVxWqnfrcHPSCnp7Oo5yNXHId9Av2Q== + +p-map@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175" + integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw== + +p-map@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/p-map/-/p-map-3.0.0.tgz#d704d9af8a2ba684e2600d9a215983d4141a979d" + integrity sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ== + dependencies: + aggregate-error "^3.0.0" + +p-map@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/p-map/-/p-map-4.0.0.tgz#bb2f95a5eda2ec168ec9274e06a747c3e2904d2b" + integrity sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ== + dependencies: + aggregate-error "^3.0.0" + +p-pipe@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/p-pipe/-/p-pipe-3.1.0.tgz#48b57c922aa2e1af6a6404cb7c6bf0eb9cc8e60e" + integrity sha512-08pj8ATpzMR0Y80x50yJHn37NF6vjrqHutASaX5LiH5npS9XPvrUmscd9MF5R4fuYRHOxQR1FfMIlF7AzwoPqw== + +p-queue@^6.6.2: + version "6.6.2" + resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.6.2.tgz#2068a9dcf8e67dd0ec3e7a2bcb76810faa85e426" + integrity sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ== + dependencies: + eventemitter3 "^4.0.4" + p-timeout "^3.2.0" + +p-reduce@^2.0.0, p-reduce@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/p-reduce/-/p-reduce-2.1.0.tgz#09408da49507c6c274faa31f28df334bc712b64a" + integrity sha512-2USApvnsutq8uoxZBGbbWM0JIYLiEMJ9RlaN7fAzVNb9OZN0SHjjTTfIcb667XynS5Y1VhwDJVDa72TnPzAYWw== + +p-timeout@^3.1.0, p-timeout@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe" + integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg== + dependencies: + p-finally "^1.0.0" + +p-try@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" + integrity sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M= + +p-try@^2.0.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" + integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== + +p-waterfall@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/p-waterfall/-/p-waterfall-2.1.1.tgz#63153a774f472ccdc4eb281cdb2967fcf158b2ee" + integrity sha512-RRTnDb2TBG/epPRI2yYXsimO0v3BXC8Yd3ogr1545IaqKK17VGhbWVeGGN+XfCm/08OK8635nH31c8bATkHuSw== + dependencies: + p-reduce "^2.0.0" + +pacote@^11.2.6: + version "11.3.5" + resolved "https://registry.yarnpkg.com/pacote/-/pacote-11.3.5.tgz#73cf1fc3772b533f575e39efa96c50be8c3dc9d2" + integrity sha512-fT375Yczn4zi+6Hkk2TBe1x1sP8FgFsEIZ2/iWaXY2r/NkhDJfxbcn5paz1+RTFCyNf+dPnaoBDJoAxXSU8Bkg== + dependencies: + "@npmcli/git" "^2.1.0" + "@npmcli/installed-package-contents" "^1.0.6" + "@npmcli/promise-spawn" "^1.2.0" + "@npmcli/run-script" "^1.8.2" + cacache "^15.0.5" + chownr "^2.0.0" + fs-minipass "^2.1.0" + infer-owner "^1.0.4" + minipass "^3.1.3" + mkdirp "^1.0.3" + npm-package-arg "^8.0.1" + npm-packlist "^2.1.4" + npm-pick-manifest "^6.0.0" + npm-registry-fetch "^11.0.0" + promise-retry "^2.0.1" + read-package-json-fast "^2.0.1" + rimraf "^3.0.2" + ssri "^8.0.1" + tar "^6.1.0" + +pad-left@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/pad-left/-/pad-left-2.1.0.tgz#16e6a3b2d44a8e138cb0838cc7cb403a4fc9e994" + integrity sha1-FuajstRKjhOMsIOMx8tAOk/J6ZQ= + dependencies: + repeat-string "^1.5.4" + +parent-module@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" + integrity sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g== + dependencies: + callsites "^3.0.0" + +parse-filepath@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/parse-filepath/-/parse-filepath-1.0.2.tgz#a632127f53aaf3d15876f5872f3ffac763d6c891" + integrity sha1-pjISf1Oq89FYdvWHLz/6x2PWyJE= + dependencies: + is-absolute "^1.0.0" + map-cache "^0.2.0" + path-root "^0.1.1" + +parse-json@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-2.2.0.tgz#f480f40434ef80741f8469099f8dea18f55a4dc9" + integrity sha1-9ID0BDTvgHQfhGkJn43qGPVaTck= + dependencies: + error-ex "^1.2.0" + +parse-json@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-4.0.0.tgz#be35f5425be1f7f6c747184f98a788cb99477ee0" + integrity sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA= + dependencies: + error-ex "^1.3.1" + json-parse-better-errors "^1.0.1" + +parse-json@^5.0.0, parse-json@^5.2.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-5.2.0.tgz#c76fc66dee54231c962b22bcc8a72cf2f99753cd" + integrity sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg== + dependencies: + "@babel/code-frame" "^7.0.0" + error-ex "^1.3.1" + json-parse-even-better-errors "^2.3.0" + lines-and-columns "^1.1.6" + +parse-node-version@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/parse-node-version/-/parse-node-version-1.0.1.tgz#e2b5dbede00e7fa9bc363607f53327e8b073189b" + integrity sha512-3YHlOa/JgH6Mnpr05jP9eDG254US9ek25LyIxZlDItp2iJtwyaXQb57lBYLdT3MowkUFYEV2XXNAYIPlESvJlA== + +parse-passwd@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/parse-passwd/-/parse-passwd-1.0.0.tgz#6d5b934a456993b23d37f40a382d6f1666a8e5c6" + integrity sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY= + +parse-path@^4.0.0: + version "4.0.3" + resolved "https://registry.yarnpkg.com/parse-path/-/parse-path-4.0.3.tgz#82d81ec3e071dcc4ab49aa9f2c9c0b8966bb22bf" + integrity sha512-9Cepbp2asKnWTJ9x2kpw6Fe8y9JDbqwahGCTvklzd/cEq5C5JC59x2Xb0Kx+x0QZ8bvNquGO8/BWP0cwBHzSAA== + dependencies: + is-ssh "^1.3.0" + protocols "^1.4.0" + qs "^6.9.4" + query-string "^6.13.8" + +parse-url@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/parse-url/-/parse-url-6.0.0.tgz#f5dd262a7de9ec00914939220410b66cff09107d" + integrity sha512-cYyojeX7yIIwuJzledIHeLUBVJ6COVLeT4eF+2P6aKVzwvgKQPndCBv3+yQ7pcWjqToYwaligxzSYNNmGoMAvw== + dependencies: + is-ssh "^1.3.0" + normalize-url "^6.1.0" + parse-path "^4.0.0" + protocols "^1.4.0" + +parse5@6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/parse5/-/parse5-6.0.1.tgz#e1a1c085c569b3dc08321184f19a39cc27f7c30b" + integrity sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw== + +pascalcase@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/pascalcase/-/pascalcase-0.1.1.tgz#b363e55e8006ca6fe21784d2db22bd15d7917f14" + integrity sha1-s2PlXoAGym/iF4TS2yK9FdeRfxQ= + +path-dirname@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/path-dirname/-/path-dirname-1.0.2.tgz#cc33d24d525e099a5388c0336c6e32b9160609e0" + integrity sha1-zDPSTVJeCZpTiMAzbG4yuRYGCeA= + +path-exists@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-2.1.0.tgz#0feb6c64f0fc518d9a754dd5efb62c7022761f4b" + integrity sha1-D+tsZPD8UY2adU3V77YscCJ2H0s= + dependencies: + pinkie-promise "^2.0.0" + +path-exists@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515" + integrity sha1-zg6+ql94yxiSXqfYENe1mwEP1RU= + +path-exists@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" + integrity sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w== + +path-is-absolute@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" + integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= + +path-key@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40" + integrity sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A= + +path-key@^3.0.0, path-key@^3.1.0: + version "3.1.1" + resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" + integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== + +path-parse@^1.0.6: + version "1.0.7" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735" + integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== + +path-root-regex@^0.1.0: + version "0.1.2" + resolved "https://registry.yarnpkg.com/path-root-regex/-/path-root-regex-0.1.2.tgz#bfccdc8df5b12dc52c8b43ec38d18d72c04ba96d" + integrity sha1-v8zcjfWxLcUsi0PsONGNcsBLqW0= + +path-root@^0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/path-root/-/path-root-0.1.1.tgz#9a4a6814cac1c0cd73360a95f32083c8ea4745b7" + integrity sha1-mkpoFMrBwM1zNgqV8yCDyOpHRbc= + dependencies: + path-root-regex "^0.1.0" + +path-type@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441" + integrity sha1-WcRPfuSR2nBNpBXaWkBwuk+P5EE= + dependencies: + graceful-fs "^4.1.2" + pify "^2.0.0" + pinkie-promise "^2.0.0" + +path-type@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/path-type/-/path-type-3.0.0.tgz#cef31dc8e0a1a3bb0d105c0cd97cf3bf47f4e36f" + integrity sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg== + dependencies: + pify "^3.0.0" + +path-type@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" + integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== + +performance-now@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" + integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= + +picomatch@^2.0.4, picomatch@^2.2.3: + version "2.3.0" + resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.0.tgz#f1f061de8f6a4bf022892e2d128234fb98302972" + integrity sha512-lY1Q/PiJGC2zOv/z391WOTD+Z02bCgsFfvxoXXf6h7kv9o+WmsmzYqrAwY63sNgOxE4xEdq0WyUnXfKeBrSvYw== + +pidtree@^0.3.0: + version "0.3.1" + resolved "https://registry.yarnpkg.com/pidtree/-/pidtree-0.3.1.tgz#ef09ac2cc0533df1f3250ccf2c4d366b0d12114a" + integrity sha512-qQbW94hLHEqCg7nhby4yRC7G2+jYHY4Rguc2bjw7Uug4GIJuu1tvf2uHaZv5Q8zdt+WKJ6qK1FOI6amaWUo5FA== + +pify@^2.0.0, pify@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" + integrity sha1-7RQaasBDqEnqWISY59yosVMw6Qw= + +pify@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/pify/-/pify-3.0.0.tgz#e5a4acd2c101fdf3d9a4d07f0dbc4db49dd28176" + integrity sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY= + +pify@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/pify/-/pify-4.0.1.tgz#4b2cd25c50d598735c50292224fd8c6df41e3231" + integrity sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g== + +pify@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/pify/-/pify-5.0.0.tgz#1f5eca3f5e87ebec28cc6d54a0e4aaf00acc127f" + integrity sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA== + +pinkie-promise@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/pinkie-promise/-/pinkie-promise-2.0.1.tgz#2135d6dfa7a358c069ac9b178776288228450ffa" + integrity sha1-ITXW36ejWMBprJsXh3YogihFD/o= + dependencies: + pinkie "^2.0.0" + +pinkie@^2.0.0: + version "2.0.4" + resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870" + integrity sha1-clVrgM+g1IqXToDnckjoDtT3+HA= + +pirates@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.1.tgz#643a92caf894566f91b2b986d2c66950a8e2fb87" + integrity sha512-WuNqLTbMI3tmfef2TKxlQmAiLHKtFhlsCZnPIpuv2Ow0RDVO8lfy1Opf4NUzlMXLjPl+Men7AuVdX6TA+s+uGA== + dependencies: + node-modules-regexp "^1.0.0" + +pkg-dir@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3" + integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ== + dependencies: + find-up "^4.0.0" + +platform@^1.3.3: + version "1.3.6" + resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.6.tgz#48b4ce983164b209c2d45a107adb31f473a6e7a7" + integrity sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg== + +plugin-error@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/plugin-error/-/plugin-error-1.0.1.tgz#77016bd8919d0ac377fdcdd0322328953ca5781c" + integrity sha512-L1zP0dk7vGweZME2i+EeakvUNqSrdiI3F91TwEoYiGrAfUXmVv6fJIq4g82PAXxNsWOp0J7ZqQy/3Szz0ajTxA== + dependencies: + ansi-colors "^1.0.1" + arr-diff "^4.0.0" + arr-union "^3.1.0" + extend-shallow "^3.0.2" + +posix-character-classes@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab" + integrity sha1-AerA/jta9xoqbAL+q7jB/vfgDqs= + +postcss@^7.0.16: + version "7.0.36" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-7.0.36.tgz#056f8cffa939662a8f5905950c07d5285644dfcb" + integrity sha512-BebJSIUMwJHRH0HAQoxN4u1CN86glsrwsW0q7T+/m44eXOUAxSNdHRkNZPYz5vVUbg17hFgOQDE7fZk7li3pZw== + dependencies: + chalk "^2.4.2" + source-map "^0.6.1" + supports-color "^6.1.0" + +prelude-ls@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" + integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g== + +prelude-ls@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" + integrity sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ= + +prettier@^2.1.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.3.2.tgz#ef280a05ec253712e486233db5c6f23441e7342d" + integrity sha512-lnJzDfJ66zkMy58OL5/NY5zp70S7Nz6KqcKkXYzn2tMVrNxvbqaBpg7H3qHaLxCJ5lNMsGuM8+ohS7cZrthdLQ== + +pretty-format@^26.0.0, pretty-format@^26.6.2: + version "26.6.2" + resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-26.6.2.tgz#e35c2705f14cb7fe2fe94fa078345b444120fc93" + integrity sha512-7AeGuCYNGmycyQbCqd/3PWH4eOoX/OiCa0uphp57NVTeAGdJGaAliecxwBDHYQCIvrW7aDBZCYeNTP/WX69mkg== + dependencies: + "@jest/types" "^26.6.2" + ansi-regex "^5.0.0" + ansi-styles "^4.0.0" + react-is "^17.0.1" + +pretty-format@^27.0.6: + version "27.0.6" + resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-27.0.6.tgz#ab770c47b2c6f893a21aefc57b75da63ef49a11f" + integrity sha512-8tGD7gBIENgzqA+UBzObyWqQ5B778VIFZA/S66cclyd5YkFLYs2Js7gxDKf0MXtTc9zcS7t1xhdfcElJ3YIvkQ== + dependencies: + "@jest/types" "^27.0.6" + ansi-regex "^5.0.0" + ansi-styles "^5.0.0" + react-is "^17.0.1" + +pretty-hrtime@^1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz#b7e3ea42435a4c9b2759d99e0f201eb195802ee1" + integrity sha1-t+PqQkNaTJsnWdmeDyAesZWALuE= + +process-nextick-args@^2.0.0, process-nextick-args@~2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" + integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== + +progress@^2.0.0, progress@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" + integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== + +promise-inflight@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3" + integrity sha1-mEcocL8igTL8vdhoEputEsPAKeM= + +promise-retry@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/promise-retry/-/promise-retry-2.0.1.tgz#ff747a13620ab57ba688f5fc67855410c370da22" + integrity sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g== + dependencies: + err-code "^2.0.2" + retry "^0.12.0" + +promise@^8.0.1: + version "8.1.0" + resolved "https://registry.yarnpkg.com/promise/-/promise-8.1.0.tgz#697c25c3dfe7435dd79fcd58c38a135888eaf05e" + integrity sha512-W04AqnILOL/sPRXziNicCjSNRruLAuIHEOVBazepu0545DDNGYHz7ar9ZgZ1fMU8/MA4mVxp5rkBWRi6OXIy3Q== + dependencies: + asap "~2.0.6" + +prompts@^2.0.1: + version "2.4.1" + resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.1.tgz#befd3b1195ba052f9fd2fde8a486c4e82ee77f61" + integrity sha512-EQyfIuO2hPDsX1L/blblV+H7I0knhgAd82cVneCwcdND9B8AuCDuRcBH6yIcG4dFzlOUqbazQqwGjx5xmsNLuQ== + dependencies: + kleur "^3.0.3" + sisteransi "^1.0.5" + +promzard@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/promzard/-/promzard-0.3.0.tgz#26a5d6ee8c7dee4cb12208305acfb93ba382a9ee" + integrity sha1-JqXW7ox97kyxIggwWs+5O6OCqe4= + dependencies: + read "1" + +proto-list@~1.2.1: + version "1.2.4" + resolved "https://registry.yarnpkg.com/proto-list/-/proto-list-1.2.4.tgz#212d5bfe1318306a420f6402b8e26ff39647a849" + integrity sha1-IS1b/hMYMGpCD2QCuOJv85ZHqEk= + +protocols@^1.1.0, protocols@^1.4.0: + version "1.4.8" + resolved "https://registry.yarnpkg.com/protocols/-/protocols-1.4.8.tgz#48eea2d8f58d9644a4a32caae5d5db290a075ce8" + integrity sha512-IgjKyaUSjsROSO8/D49Ab7hP8mJgTYcqApOqdPhLoPxAplXmkp+zRvsrSQjFn5by0rhm4VH0GAUELIPpx7B1yg== + +psl@^1.1.28, psl@^1.1.33: + version "1.8.0" + resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" + integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== + +pump@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/pump/-/pump-2.0.1.tgz#12399add6e4cf7526d973cbc8b5ce2e2908b3909" + integrity sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA== + dependencies: + end-of-stream "^1.1.0" + once "^1.3.1" + +pumpify@^1.3.5: + version "1.5.1" + resolved "https://registry.yarnpkg.com/pumpify/-/pumpify-1.5.1.tgz#36513be246ab27570b1a374a5ce278bfd74370ce" + integrity sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ== + dependencies: + duplexify "^3.6.0" + inherits "^2.0.3" + pump "^2.0.0" + +punycode@^2.1.0, punycode@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" + integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== + +q@^1.5.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" + integrity sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc= + +qs@^6.9.4: + version "6.10.1" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.10.1.tgz#4931482fa8d647a5aab799c5271d2133b981fb6a" + integrity sha512-M528Hph6wsSVOBiYUnGf+K/7w0hNshs/duGsNXPUCLH5XAqjEtiPGwNONLV0tBH8NoGb0mvD5JubnUTrujKDTg== + dependencies: + side-channel "^1.0.4" + +qs@~6.5.2: + version "6.5.2" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" + integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA== + +query-string@^6.13.8: + version "6.14.1" + resolved "https://registry.yarnpkg.com/query-string/-/query-string-6.14.1.tgz#7ac2dca46da7f309449ba0f86b1fd28255b0c86a" + integrity sha512-XDxAeVmpfu1/6IjyT/gXHOl+S0vQ9owggJ30hhWKdHAsNPOcasn5o9BW0eejZqL2e4vMjhAxoW3jVHcD6mbcYw== + dependencies: + decode-uri-component "^0.2.0" + filter-obj "^1.1.0" + split-on-first "^1.0.0" + strict-uri-encode "^2.0.0" + +queue-microtask@^1.2.2: + version "1.2.3" + resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" + integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== + +quick-lru@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-4.0.1.tgz#5b8878f113a58217848c6482026c73e1ba57727f" + integrity sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g== + +quick-lru@^5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/quick-lru/-/quick-lru-5.1.1.tgz#366493e6b3e42a3a6885e2e99d18f80fb7a8c932" + integrity sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA== + +randomatic@3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/randomatic/-/randomatic-3.1.1.tgz#b776efc59375984e36c537b2f51a1f0aff0da1ed" + integrity sha512-TuDE5KxZ0J461RVjrJZCJc+J+zCkTb1MbH9AQUq68sMhOMcy9jLcb3BrZKgp9q9Ncltdg4QVqWrH02W2EFFVYw== + dependencies: + is-number "^4.0.0" + kind-of "^6.0.0" + math-random "^1.0.1" + +react-is@^17.0.1: + version "17.0.2" + resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0" + integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w== + +read-cmd-shim@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/read-cmd-shim/-/read-cmd-shim-2.0.0.tgz#4a50a71d6f0965364938e9038476f7eede3928d9" + integrity sha512-HJpV9bQpkl6KwjxlJcBoqu9Ba0PQg8TqSNIOrulGt54a0uup0HtevreFHzYzkm0lpnleRdNBzXznKrgxglEHQw== + +read-package-json-fast@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/read-package-json-fast/-/read-package-json-fast-2.0.2.tgz#2dcb24d9e8dd50fb322042c8c35a954e6cc7ac9e" + integrity sha512-5fyFUyO9B799foVk4n6ylcoAktG/FbE3jwRKxvwaeSrIunaoMc0u81dzXxjeAFKOce7O5KncdfwpGvvs6r5PsQ== + dependencies: + json-parse-even-better-errors "^2.3.0" + npm-normalize-package-bin "^1.0.1" + +read-package-json@^2.0.0: + version "2.1.2" + resolved "https://registry.yarnpkg.com/read-package-json/-/read-package-json-2.1.2.tgz#6992b2b66c7177259feb8eaac73c3acd28b9222a" + integrity sha512-D1KmuLQr6ZSJS0tW8hf3WGpRlwszJOXZ3E8Yd/DNRaM5d+1wVRZdHlpGBLAuovjr28LbWvjpWkBHMxpRGGjzNA== + dependencies: + glob "^7.1.1" + json-parse-even-better-errors "^2.3.0" + normalize-package-data "^2.0.0" + npm-normalize-package-bin "^1.0.0" + +read-package-json@^3.0.0, read-package-json@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/read-package-json/-/read-package-json-3.0.1.tgz#c7108f0b9390257b08c21e3004d2404c806744b9" + integrity sha512-aLcPqxovhJTVJcsnROuuzQvv6oziQx4zd3JvG0vGCL5MjTONUc4uJ90zCBC6R7W7oUKBNoR/F8pkyfVwlbxqng== + dependencies: + glob "^7.1.1" + json-parse-even-better-errors "^2.3.0" + normalize-package-data "^3.0.0" + npm-normalize-package-bin "^1.0.0" + +read-package-tree@^5.3.1: + version "5.3.1" + resolved "https://registry.yarnpkg.com/read-package-tree/-/read-package-tree-5.3.1.tgz#a32cb64c7f31eb8a6f31ef06f9cedf74068fe636" + integrity sha512-mLUDsD5JVtlZxjSlPPx1RETkNjjvQYuweKwNVt1Sn8kP5Jh44pvYuUHCp6xSVDZWbNxVxG5lyZJ921aJH61sTw== + dependencies: + read-package-json "^2.0.0" + readdir-scoped-modules "^1.0.0" + util-promisify "^2.1.0" + +read-pkg-up@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-1.0.1.tgz#9d63c13276c065918d57f002a57f40a1b643fb02" + integrity sha1-nWPBMnbAZZGNV/ACpX9AobZD+wI= + dependencies: + find-up "^1.0.0" + read-pkg "^1.0.0" + +read-pkg-up@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-3.0.0.tgz#3ed496685dba0f8fe118d0691dc51f4a1ff96f07" + integrity sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc= + dependencies: + find-up "^2.0.0" + read-pkg "^3.0.0" + +read-pkg-up@^7.0.1: + version "7.0.1" + resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-7.0.1.tgz#f3a6135758459733ae2b95638056e1854e7ef507" + integrity sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg== + dependencies: + find-up "^4.1.0" + read-pkg "^5.2.0" + type-fest "^0.8.1" + +read-pkg-up@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-8.0.0.tgz#72f595b65e66110f43b052dd9af4de6b10534670" + integrity sha512-snVCqPczksT0HS2EC+SxUndvSzn6LRCwpfSvLrIfR5BKDQQZMaI6jPRC9dYvYFDRAuFEAnkwww8kBBNE/3VvzQ== + dependencies: + find-up "^5.0.0" + read-pkg "^6.0.0" + type-fest "^1.0.1" + +read-pkg@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-1.1.0.tgz#f5ffaa5ecd29cb31c0474bca7d756b6bb29e3f28" + integrity sha1-9f+qXs0pyzHAR0vKfXVra7KePyg= + dependencies: + load-json-file "^1.0.0" + normalize-package-data "^2.3.2" + path-type "^1.0.0" + +read-pkg@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-3.0.0.tgz#9cbc686978fee65d16c00e2b19c237fcf6e38389" + integrity sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k= + dependencies: + load-json-file "^4.0.0" + normalize-package-data "^2.3.2" + path-type "^3.0.0" + +read-pkg@^5.2.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-5.2.0.tgz#7bf295438ca5a33e56cd30e053b34ee7250c93cc" + integrity sha512-Ug69mNOpfvKDAc2Q8DRpMjjzdtrnv9HcSMX+4VsZxD1aZ6ZzrIE7rlzXBtWTyhULSMKg076AW6WR5iZpD0JiOg== + dependencies: + "@types/normalize-package-data" "^2.4.0" + normalize-package-data "^2.5.0" + parse-json "^5.0.0" + type-fest "^0.6.0" + +read-pkg@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-6.0.0.tgz#a67a7d6a1c2b0c3cd6aa2ea521f40c458a4a504c" + integrity sha512-X1Fu3dPuk/8ZLsMhEj5f4wFAF0DWoK7qhGJvgaijocXxBmSToKfbFtqbxMO7bVjNA1dmE5huAzjXj/ey86iw9Q== + dependencies: + "@types/normalize-package-data" "^2.4.0" + normalize-package-data "^3.0.2" + parse-json "^5.2.0" + type-fest "^1.0.1" + +read@1, read@~1.0.1: + version "1.0.7" + resolved "https://registry.yarnpkg.com/read/-/read-1.0.7.tgz#b3da19bd052431a97671d44a42634adf710b40c4" + integrity sha1-s9oZvQUkMal2cdRKQmNK33ELQMQ= + dependencies: + mute-stream "~0.0.4" + +"readable-stream@2 || 3", readable-stream@3, readable-stream@^3.0.0, readable-stream@^3.0.2, readable-stream@^3.6.0: + version "3.6.0" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" + integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== + dependencies: + inherits "^2.0.3" + string_decoder "^1.1.1" + util-deprecate "^1.0.1" + +readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.3.3, readable-stream@^2.3.5, readable-stream@^2.3.6, readable-stream@~2.3.6: + version "2.3.7" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.7.tgz#1eca1cf711aef814c04f62252a36a62f6cb23b57" + integrity sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + +readdir-scoped-modules@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/readdir-scoped-modules/-/readdir-scoped-modules-1.1.0.tgz#8d45407b4f870a0dcaebc0e28670d18e74514309" + integrity sha512-asaikDeqAQg7JifRsZn1NJZXo9E+VwlyCfbkZhwyISinqk5zNS6266HS5kah6P0SaQKGF6SkNnZVHUzHFYxYDw== + dependencies: + debuglog "^1.0.1" + dezalgo "^1.0.0" + graceful-fs "^4.1.2" + once "^1.3.0" + +readdirp@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-2.2.1.tgz#0e87622a3325aa33e892285caf8b4e846529a525" + integrity sha512-1JU/8q+VgFZyxwrJ+SVIOsh+KywWGpds3NTqikiKpDMZWScmAYyKIgqkO+ARvNWJfXeXR1zxz7aHF4u4CyH6vQ== + dependencies: + graceful-fs "^4.1.11" + micromatch "^3.1.10" + readable-stream "^2.0.2" + +rechoir@^0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384" + integrity sha1-hSBLVNuoLVdC4oyWdW70OvUOM4Q= + dependencies: + resolve "^1.1.6" + +redent@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/redent/-/redent-3.0.0.tgz#e557b7998316bb53c9f1f56fa626352c6963059f" + integrity sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg== + dependencies: + indent-string "^4.0.0" + strip-indent "^3.0.0" + +redent@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/redent/-/redent-4.0.0.tgz#0c0ba7caabb24257ab3bb7a4fd95dd1d5c5681f9" + integrity sha512-tYkDkVVtYkSVhuQ4zBgfvciymHaeuel+zFKXShfDnFP5SyVEP7qo70Rf1jTOTCx3vGNAbnEi/xFkcfQVMIBWag== + dependencies: + indent-string "^5.0.0" + strip-indent "^4.0.0" + +reduce-flatten@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/reduce-flatten/-/reduce-flatten-2.0.0.tgz#734fd84e65f375d7ca4465c69798c25c9d10ae27" + integrity sha512-EJ4UNY/U1t2P/2k6oqotuX2Cc3T6nxJwsM0N0asT7dhrtH1ltUxDn4NalSYmPE2rCkVpcf/X6R0wDwcFpzhd4w== + +regex-not@^1.0.0, regex-not@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/regex-not/-/regex-not-1.0.2.tgz#1f4ece27e00b0b65e0247a6810e6a85d83a5752c" + integrity sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A== + dependencies: + extend-shallow "^3.0.2" + safe-regex "^1.1.0" + +regexpp@^3.1.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.2.0.tgz#0425a2768d8f23bad70ca4b90461fa2f1213e1b2" + integrity sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg== + +remove-bom-buffer@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/remove-bom-buffer/-/remove-bom-buffer-3.0.0.tgz#c2bf1e377520d324f623892e33c10cac2c252b53" + integrity sha512-8v2rWhaakv18qcvNeli2mZ/TMTL2nEyAKRvzo1WtnZBl15SHyEhrCu2/xKlJyUFKHiHgfXIyuY6g2dObJJycXQ== + dependencies: + is-buffer "^1.1.5" + is-utf8 "^0.2.1" + +remove-bom-stream@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/remove-bom-stream/-/remove-bom-stream-1.2.0.tgz#05f1a593f16e42e1fb90ebf59de8e569525f9523" + integrity sha1-BfGlk/FuQuH7kOv1nejlaVJflSM= + dependencies: + remove-bom-buffer "^3.0.0" + safe-buffer "^5.1.0" + through2 "^2.0.3" + +remove-trailing-separator@^1.0.1, remove-trailing-separator@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz#c24bce2a283adad5bc3f58e0d48249b92379d8ef" + integrity sha1-wkvOKig62tW8P1jg1IJJuSN52O8= + +repeat-element@^1.1.2: + version "1.1.4" + resolved "https://registry.yarnpkg.com/repeat-element/-/repeat-element-1.1.4.tgz#be681520847ab58c7568ac75fbfad28ed42d39e9" + integrity sha512-LFiNfRcSu7KK3evMyYOuCzv3L10TW7yC1G2/+StMjK8Y6Vqd2MG7r/Qjw4ghtuCOjFvlnms/iMmLqpvW/ES/WQ== + +repeat-string@^1.5.4, repeat-string@^1.6.1: + version "1.6.1" + resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" + integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc= + +replace-ext@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/replace-ext/-/replace-ext-1.0.1.tgz#2d6d996d04a15855d967443631dd5f77825b016a" + integrity sha512-yD5BHCe7quCgBph4rMQ+0KkIRKwWCrHDOX1p1Gp6HwjPM5kVoCdKGNhN7ydqqsX6lJEnQDKZ/tFMiEdQ1dvPEw== + +replace-homedir@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/replace-homedir/-/replace-homedir-1.0.0.tgz#e87f6d513b928dde808260c12be7fec6ff6e798c" + integrity sha1-6H9tUTuSjd6AgmDBK+f+xv9ueYw= + dependencies: + homedir-polyfill "^1.0.1" + is-absolute "^1.0.0" + remove-trailing-separator "^1.1.0" + +request@^2.88.0, request@^2.88.2: + version "2.88.2" + resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" + integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== + dependencies: + aws-sign2 "~0.7.0" + aws4 "^1.8.0" + caseless "~0.12.0" + combined-stream "~1.0.6" + extend "~3.0.2" + forever-agent "~0.6.1" + form-data "~2.3.2" + har-validator "~5.1.3" + http-signature "~1.2.0" + is-typedarray "~1.0.0" + isstream "~0.1.2" + json-stringify-safe "~5.0.1" + mime-types "~2.1.19" + oauth-sign "~0.9.0" + performance-now "^2.1.0" + qs "~6.5.2" + safe-buffer "^5.1.2" + tough-cookie "~2.5.0" + tunnel-agent "^0.6.0" + uuid "^3.3.2" + +require-directory@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" + integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I= + +require-from-string@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909" + integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw== + +require-main-filename@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-1.0.1.tgz#97f717b69d48784f5f526a6c5aa8ffdda055a4d1" + integrity sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE= + +resolve-cwd@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/resolve-cwd/-/resolve-cwd-3.0.0.tgz#0f0075f1bb2544766cf73ba6a6e2adfebcb13f2d" + integrity sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg== + dependencies: + resolve-from "^5.0.0" + +resolve-dir@^1.0.0, resolve-dir@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/resolve-dir/-/resolve-dir-1.0.1.tgz#79a40644c362be82f26effe739c9bb5382046f43" + integrity sha1-eaQGRMNivoLybv/nOcm7U4IEb0M= + dependencies: + expand-tilde "^2.0.0" + global-modules "^1.0.0" + +resolve-from@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" + integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== + +resolve-from@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-5.0.0.tgz#c35225843df8f776df21c57557bc087e9dfdfc69" + integrity sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw== + +resolve-options@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/resolve-options/-/resolve-options-1.1.0.tgz#32bb9e39c06d67338dc9378c0d6d6074566ad131" + integrity sha1-MrueOcBtZzONyTeMDW1gdFZq0TE= + dependencies: + value-or-function "^3.0.0" + +resolve-url@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a" + integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo= + +resolve@^1.1.6, resolve@^1.1.7, resolve@^1.10.0, resolve@^1.20.0, resolve@^1.4.0: + version "1.20.0" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.20.0.tgz#629a013fb3f70755d6f0b7935cc1c2c5378b1975" + integrity sha512-wENBPt4ySzg4ybFQW2TT1zMQucPK95HSh/nq2CFTZVOGut2+pQvSsgtda4d26YrYcr067wjbmzOG8byDPBX63A== + dependencies: + is-core-module "^2.2.0" + path-parse "^1.0.6" + +restore-cursor@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-3.1.0.tgz#39f67c54b3a7a58cea5236d95cf0034239631f7e" + integrity sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA== + dependencies: + onetime "^5.1.0" + signal-exit "^3.0.2" + +ret@~0.1.10: + version "0.1.15" + resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc" + integrity sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg== + +retry@^0.12.0: + version "0.12.0" + resolved "https://registry.yarnpkg.com/retry/-/retry-0.12.0.tgz#1b42a6266a21f07421d1b0b54b7dc167b01c013b" + integrity sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs= + +reusify@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/reusify/-/reusify-1.0.4.tgz#90da382b1e126efc02146e90845a88db12925d76" + integrity sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw== + +rimraf@^2.6.3: + version "2.7.1" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" + integrity sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w== + dependencies: + glob "^7.1.3" + +rimraf@^3.0.0, rimraf@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-3.0.2.tgz#f1a5402ba6220ad52cc1282bac1ae3aa49fd061a" + integrity sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA== + dependencies: + glob "^7.1.3" + +run-async@^2.4.0: + version "2.4.1" + resolved "https://registry.yarnpkg.com/run-async/-/run-async-2.4.1.tgz#8440eccf99ea3e70bd409d49aab88e10c189a455" + integrity sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ== + +run-parallel@^1.1.9: + version "1.2.0" + resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee" + integrity sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA== + dependencies: + queue-microtask "^1.2.2" + +rxjs@7.2.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-7.2.0.tgz#5cd12409639e9514a71c9f5f9192b2c4ae94de31" + integrity sha512-aX8w9OpKrQmiPKfT1bqETtUr9JygIz6GZ+gql8v7CijClsP0laoFUdKzxFAoWuRdSlOdU2+crss+cMf+cqMTnw== + dependencies: + tslib "~2.1.0" + +rxjs@^6.6.0: + version "6.6.7" + resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.6.7.tgz#90ac018acabf491bf65044235d5863c4dab804c9" + integrity sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ== + dependencies: + tslib "^1.9.0" + +safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.2, safe-buffer@~5.2.0: + version "5.2.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" + integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== + +safe-buffer@~5.1.0, safe-buffer@~5.1.1: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== + +safe-regex@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/safe-regex/-/safe-regex-1.1.0.tgz#40a3669f3b077d1e943d44629e157dd48023bf2e" + integrity sha1-QKNmnzsHfR6UPURinhV91IAjvy4= + dependencies: + ret "~0.1.10" + +"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: + version "2.1.2" + resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" + integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== + +sax@>=0.6.0: + version "1.2.4" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" + integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== + +saxes@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/saxes/-/saxes-5.0.1.tgz#eebab953fa3b7608dbe94e5dadb15c888fa6696d" + integrity sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw== + dependencies: + xmlchars "^2.2.0" + +semver-greatest-satisfied-range@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/semver-greatest-satisfied-range/-/semver-greatest-satisfied-range-1.1.0.tgz#13e8c2658ab9691cb0cd71093240280d36f77a5b" + integrity sha1-E+jCZYq5aRywzXEJMkAoDTb3els= + dependencies: + sver-compat "^1.5.0" + +"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: + version "5.7.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" + integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== + +semver@7.x, semver@^7.1.1, semver@^7.1.3, semver@^7.2.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5: + version "7.3.5" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.5.tgz#0b621c879348d8998e4b0e4be94b3f12e6018ef7" + integrity sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ== + dependencies: + lru-cache "^6.0.0" + +semver@^6.0.0, semver@^6.3.0: + version "6.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" + integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== + +set-blocking@^2.0.0, set-blocking@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" + integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc= + +set-value@^2.0.0, set-value@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/set-value/-/set-value-2.0.1.tgz#a18d40530e6f07de4228c7defe4227af8cad005b" + integrity sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw== + dependencies: + extend-shallow "^2.0.1" + is-extendable "^0.1.1" + is-plain-object "^2.0.3" + split-string "^3.0.1" + +shallow-clone@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-3.0.1.tgz#8f2981ad92531f55035b01fb230769a40e02efa3" + integrity sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA== + dependencies: + kind-of "^6.0.2" + +shebang-command@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea" + integrity sha1-RKrGW2lbAzmJaMOfNj/uXer98eo= + dependencies: + shebang-regex "^1.0.0" + +shebang-command@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea" + integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA== + dependencies: + shebang-regex "^3.0.0" + +shebang-regex@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3" + integrity sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM= + +shebang-regex@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" + integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== + +shell-quote@^1.6.1: + version "1.7.2" + resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.7.2.tgz#67a7d02c76c9da24f99d20808fcaded0e0e04be2" + integrity sha512-mRz/m/JVscCrkMyPqHc/bczi3OQHkLTqXHEFu0zDhK/qfv3UcOA4SVmRCLmos4bhjr9ekVQubj/R7waKapmiQg== + +shiki@^0.9.3: + version "0.9.5" + resolved "https://registry.yarnpkg.com/shiki/-/shiki-0.9.5.tgz#c8da81a05fbfd1810729c6873901a729a72ec541" + integrity sha512-XFn+rl3wIowDjzdr5DlHoHgQphXefgUTs2bNp/bZu4WF9gTrTLnKwio3f28VjiFG6Jpip7yQn/p4mMj6OrjrtQ== + dependencies: + json5 "^2.2.0" + onigasm "^2.2.5" + vscode-textmate "5.2.0" + +side-channel@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" + integrity sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw== + dependencies: + call-bind "^1.0.0" + get-intrinsic "^1.0.2" + object-inspect "^1.9.0" + +signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c" + integrity sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA== + +sisteransi@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.5.tgz#134d681297756437cc05ca01370d3a7a571075ed" + integrity sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg== + +slash@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/slash/-/slash-2.0.0.tgz#de552851a1759df3a8f206535442f5ec4ddeab44" + integrity sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A== + +slash@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634" + integrity sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q== + +slice-ansi@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-4.0.0.tgz#500e8dd0fd55b05815086255b3195adf2a45fe6b" + integrity sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ== + dependencies: + ansi-styles "^4.0.0" + astral-regex "^2.0.0" + is-fullwidth-code-point "^3.0.0" + +slide@^1.1.6: + version "1.1.6" + resolved "https://registry.yarnpkg.com/slide/-/slide-1.1.6.tgz#56eb027d65b4d2dce6cb2e2d32c4d4afc9e1d707" + integrity sha1-VusCfWW00tzmyy4tMsTUr8nh1wc= + +smart-buffer@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.1.0.tgz#91605c25d91652f4661ea69ccf45f1b331ca21ba" + integrity sha512-iVICrxOzCynf/SNaBQCw34eM9jROU/s5rzIhpOvzhzuYHfJR/DhZfDkXiZSgKXfgv26HT3Yni3AV/DGw0cGnnw== + +snapdragon-node@^2.0.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/snapdragon-node/-/snapdragon-node-2.1.1.tgz#6c175f86ff14bdb0724563e8f3c1b021a286853b" + integrity sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw== + dependencies: + define-property "^1.0.0" + isobject "^3.0.0" + snapdragon-util "^3.0.1" + +snapdragon-util@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/snapdragon-util/-/snapdragon-util-3.0.1.tgz#f956479486f2acd79700693f6f7b805e45ab56e2" + integrity sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ== + dependencies: + kind-of "^3.2.0" + +snapdragon@^0.8.1: + version "0.8.2" + resolved "https://registry.yarnpkg.com/snapdragon/-/snapdragon-0.8.2.tgz#64922e7c565b0e14204ba1aa7d6964278d25182d" + integrity sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg== + dependencies: + base "^0.11.1" + debug "^2.2.0" + define-property "^0.2.5" + extend-shallow "^2.0.1" + map-cache "^0.2.2" + source-map "^0.5.6" + source-map-resolve "^0.5.0" + use "^3.1.0" + +socks-proxy-agent@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-5.0.1.tgz#032fb583048a29ebffec2e6a73fca0761f48177e" + integrity sha512-vZdmnjb9a2Tz6WEQVIurybSwElwPxMZaIc7PzqbJTrezcKNznv6giT7J7tZDZ1BojVaa1jvO/UiUdhDVB0ACoQ== + dependencies: + agent-base "^6.0.2" + debug "4" + socks "^2.3.3" + +socks@^2.3.3: + version "2.6.1" + resolved "https://registry.yarnpkg.com/socks/-/socks-2.6.1.tgz#989e6534a07cf337deb1b1c94aaa44296520d30e" + integrity sha512-kLQ9N5ucj8uIcxrDwjm0Jsqk06xdpBjGNQtpXy4Q8/QY2k+fY7nZH8CARy+hkbG+SGAovmzzuauCpBlb8FrnBA== + dependencies: + ip "^1.1.5" + smart-buffer "^4.1.0" + +sort-keys@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/sort-keys/-/sort-keys-2.0.0.tgz#658535584861ec97d730d6cf41822e1f56684128" + integrity sha1-ZYU1WEhh7JfXMNbPQYIuH1ZoQSg= + dependencies: + is-plain-obj "^1.0.0" + +sort-keys@^4.0.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/sort-keys/-/sort-keys-4.2.0.tgz#6b7638cee42c506fff8c1cecde7376d21315be18" + integrity sha512-aUYIEU/UviqPgc8mHR6IW1EGxkAXpeRETYcrzg8cLAvUPZcpAlleSXHV2mY7G12GphSH6Gzv+4MMVSSkbdteHg== + dependencies: + is-plain-obj "^2.0.0" + +source-map-resolve@^0.5.0: + version "0.5.3" + resolved "https://registry.yarnpkg.com/source-map-resolve/-/source-map-resolve-0.5.3.tgz#190866bece7553e1f8f267a2ee82c606b5509a1a" + integrity sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw== + dependencies: + atob "^2.1.2" + decode-uri-component "^0.2.0" + resolve-url "^0.2.1" + source-map-url "^0.4.0" + urix "^0.1.0" + +source-map-resolve@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/source-map-resolve/-/source-map-resolve-0.6.0.tgz#3d9df87e236b53f16d01e58150fc7711138e5ed2" + integrity sha512-KXBr9d/fO/bWo97NXsPIAW1bFSBOuCnjbNTBMO7N59hsv5i9yzRDfcYwwt0l04+VqnKC+EwzvJZIP/qkuMgR/w== + dependencies: + atob "^2.1.2" + decode-uri-component "^0.2.0" + +source-map-support@^0.5.17, source-map-support@^0.5.6: + version "0.5.19" + resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.19.tgz#a98b62f86dcaf4f67399648c085291ab9e8fed61" + integrity sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw== + dependencies: + buffer-from "^1.0.0" + source-map "^0.6.0" + +source-map-url@^0.4.0: + version "0.4.1" + resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.1.tgz#0af66605a745a5a2f91cf1bbf8a7afbc283dec56" + integrity sha512-cPiFOTLUKvJFIg4SKVScy4ilPPW6rFgMgfuZJPNoDuMs3nC1HbMUycBoJw77xFIp6z1UJQJOfx6C9GMH80DiTw== + +source-map@^0.5.0, source-map@^0.5.1, source-map@^0.5.6: + version "0.5.7" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc" + integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w= + +source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.1: + version "0.6.1" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" + integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== + +source-map@^0.7.3: + version "0.7.3" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.7.3.tgz#5302f8169031735226544092e64981f751750383" + integrity sha512-CkCj6giN3S+n9qrYiBTX5gystlENnRW5jZeNLHpe6aue+SrHcG5VYwujhW9s4dY31mEGsxBDrHR6oI69fTXsaQ== + +sparkles@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/sparkles/-/sparkles-1.0.1.tgz#008db65edce6c50eec0c5e228e1945061dd0437c" + integrity sha512-dSO0DDYUahUt/0/pD/Is3VIm5TGJjludZ0HVymmhYF6eNA53PVLhnUk0znSYbH8IYBuJdCE+1luR22jNLMaQdw== + +spdx-correct@^3.0.0: + version "3.1.1" + resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-3.1.1.tgz#dece81ac9c1e6713e5f7d1b6f17d468fa53d89a9" + integrity sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w== + dependencies: + spdx-expression-parse "^3.0.0" + spdx-license-ids "^3.0.0" + +spdx-exceptions@^2.1.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz#3f28ce1a77a00372683eade4a433183527a2163d" + integrity sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A== + +spdx-expression-parse@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz#cf70f50482eefdc98e3ce0a6833e4a53ceeba679" + integrity sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q== + dependencies: + spdx-exceptions "^2.1.0" + spdx-license-ids "^3.0.0" + +spdx-license-ids@^3.0.0: + version "3.0.9" + resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.9.tgz#8a595135def9592bda69709474f1cbeea7c2467f" + integrity sha512-Ki212dKK4ogX+xDo4CtOZBVIwhsKBEfsEEcwmJfLQzirgc2jIWdzg40Unxz/HzEUqM1WFzVlQSMF9kZZ2HboLQ== + +split-on-first@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/split-on-first/-/split-on-first-1.1.0.tgz#f610afeee3b12bce1d0c30425e76398b78249a5f" + integrity sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw== + +split-string@^3.0.1, split-string@^3.0.2: + version "3.1.0" + resolved "https://registry.yarnpkg.com/split-string/-/split-string-3.1.0.tgz#7cb09dda3a86585705c64b39a6466038682e8fe2" + integrity sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw== + dependencies: + extend-shallow "^3.0.0" + +split2@^3.0.0: + version "3.2.2" + resolved "https://registry.yarnpkg.com/split2/-/split2-3.2.2.tgz#bf2cf2a37d838312c249c89206fd7a17dd12365f" + integrity sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg== + dependencies: + readable-stream "^3.0.0" + +split@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" + integrity sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg== + dependencies: + through "2" + +sprintf-js@~1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" + integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= + +sshpk@^1.7.0: + version "1.16.1" + resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877" + integrity sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg== + dependencies: + asn1 "~0.2.3" + assert-plus "^1.0.0" + bcrypt-pbkdf "^1.0.0" + dashdash "^1.12.0" + ecc-jsbn "~0.1.1" + getpass "^0.1.1" + jsbn "~0.1.0" + safer-buffer "^2.0.2" + tweetnacl "~0.14.0" + +ssri@^8.0.0, ssri@^8.0.1: + version "8.0.1" + resolved "https://registry.yarnpkg.com/ssri/-/ssri-8.0.1.tgz#638e4e439e2ffbd2cd289776d5ca457c4f51a2af" + integrity sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ== + dependencies: + minipass "^3.1.1" + +stack-trace@0.0.10: + version "0.0.10" + resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0" + integrity sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA= + +stack-utils@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-2.0.3.tgz#cd5f030126ff116b78ccb3c027fe302713b61277" + integrity sha512-gL//fkxfWUsIlFL2Tl42Cl6+HFALEaB1FU76I/Fy+oZjRreP7OPMXFlGbxM7NQsI0ZpUfw76sHnv0WNYuTb7Iw== + dependencies: + escape-string-regexp "^2.0.0" + +static-extend@^0.1.1: + version "0.1.2" + resolved "https://registry.yarnpkg.com/static-extend/-/static-extend-0.1.2.tgz#60809c39cbff55337226fd5e0b520f341f1fb5c6" + integrity sha1-YICcOcv/VTNyJv1eC1IPNB8ftcY= + dependencies: + define-property "^0.2.5" + object-copy "^0.1.0" + +stats-median@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/stats-median/-/stats-median-1.0.1.tgz#ca8497cb1014d23d145db4d6fc93c8e815eed3ef" + integrity sha512-IYsheLg6dasD3zT/w9+8Iq9tcIQqqu91ZIpJOnIEM25C3X/g4Tl8mhXwW2ZQpbrsJISr9+wizEYgsibN5/b32Q== + +stream-exhaust@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/stream-exhaust/-/stream-exhaust-1.0.2.tgz#acdac8da59ef2bc1e17a2c0ccf6c320d120e555d" + integrity sha512-b/qaq/GlBK5xaq1yrK9/zFcyRSTNxmcZwFLGSTG0mXgZl/4Z6GgiyYOXOvY7N3eEvFRAG1bkDRz5EPGSvPYQlw== + +stream-shift@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.1.tgz#d7088281559ab2778424279b0877da3c392d5a3d" + integrity sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ== + +strict-uri-encode@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz#b9c7330c7042862f6b142dc274bbcc5866ce3546" + integrity sha1-ucczDHBChi9rFC3CdLvMWGbONUY= + +string-length@^4.0.1: + version "4.0.2" + resolved "https://registry.yarnpkg.com/string-length/-/string-length-4.0.2.tgz#a8a8dc7bd5c1a82b9b3c8b87e125f66871b6e57a" + integrity sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ== + dependencies: + char-regex "^1.0.2" + strip-ansi "^6.0.0" + +string-width@^1.0.1, string-width@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3" + integrity sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M= + dependencies: + code-point-at "^1.0.0" + is-fullwidth-code-point "^1.0.0" + strip-ansi "^3.0.0" + +"string-width@^1.0.2 || 2": + version "2.1.1" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-2.1.1.tgz#ab93f27a8dc13d28cac815c462143a6d9012ae9e" + integrity sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw== + dependencies: + is-fullwidth-code-point "^2.0.0" + strip-ansi "^4.0.0" + +string-width@^4.1.0, string-width@^4.2.0: + version "4.2.2" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.2.tgz#dafd4f9559a7585cfba529c6a0a4f73488ebd4c5" + integrity sha512-XBJbT3N4JhVumXE0eoLU9DCjcaF92KLNqTmFCnG1pf8duUxFGwtP6AD6nkjw9a3IdiRtL3E2w3JDiE/xi3vOeA== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.0" + +string.prototype.padend@^3.0.0: + version "3.1.2" + resolved "https://registry.yarnpkg.com/string.prototype.padend/-/string.prototype.padend-3.1.2.tgz#6858ca4f35c5268ebd5e8615e1327d55f59ee311" + integrity sha512-/AQFLdYvePENU3W5rgurfWSMU6n+Ww8n/3cUt7E+vPBB/D7YDG8x+qjoFs4M/alR2bW7Qg6xMjVwWUOvuQ0XpQ== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.3" + es-abstract "^1.18.0-next.2" + +string.prototype.trimend@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz#e75ae90c2942c63504686c18b287b4a0b1a45f80" + integrity sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.3" + +string.prototype.trimstart@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz#b36399af4ab2999b4c9c648bd7a3fb2bb26feeed" + integrity sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw== + dependencies: + call-bind "^1.0.2" + define-properties "^1.1.3" + +string_decoder@^1.1.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" + integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== + dependencies: + safe-buffer "~5.2.0" + +string_decoder@~1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" + integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== + dependencies: + safe-buffer "~5.1.0" + +strip-ansi@^3.0.0, strip-ansi@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" + integrity sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8= + dependencies: + ansi-regex "^2.0.0" + +strip-ansi@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-4.0.0.tgz#a8479022eb1ac368a871389b635262c505ee368f" + integrity sha1-qEeQIusaw2iocTibY1JixQXuNo8= + dependencies: + ansi-regex "^3.0.0" + +strip-ansi@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532" + integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w== + dependencies: + ansi-regex "^5.0.0" + +strip-bom-string@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/strip-bom-string/-/strip-bom-string-1.0.0.tgz#e5211e9224369fbb81d633a2f00044dc8cedad92" + integrity sha1-5SEekiQ2n7uB1jOi8ABE3IztrZI= + +strip-bom@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz#6219a85616520491f35788bdbf1447a99c7e6b0e" + integrity sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4= + dependencies: + is-utf8 "^0.2.0" + +strip-bom@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3" + integrity sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM= + +strip-bom@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-4.0.0.tgz#9c3505c1db45bcedca3d9cf7a16f5c5aa3901878" + integrity sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w== + +strip-final-newline@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/strip-final-newline/-/strip-final-newline-2.0.0.tgz#89b852fb2fcbe936f6f4b3187afb0a12c1ab58ad" + integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA== + +strip-indent@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/strip-indent/-/strip-indent-3.0.0.tgz#c32e1cee940b6b3432c771bc2c54bcce73cd3001" + integrity sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ== + dependencies: + min-indent "^1.0.0" + +strip-indent@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/strip-indent/-/strip-indent-4.0.0.tgz#b41379433dd06f5eae805e21d631e07ee670d853" + integrity sha512-mnVSV2l+Zv6BLpSD/8V87CW/y9EmmbYzGCIavsnsI6/nwn26DwffM/yztm30Z/I2DY9wdS3vXVCMnHDgZaVNoA== + dependencies: + min-indent "^1.0.1" + +strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" + integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== + +strong-log-transformer@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/strong-log-transformer/-/strong-log-transformer-2.1.0.tgz#0f5ed78d325e0421ac6f90f7f10e691d6ae3ae10" + integrity sha512-B3Hgul+z0L9a236FAUC9iZsL+nVHgoCJnqCbN588DjYxvGXaXaaFbfmQ/JhvKjZwsOukuR72XbHv71Qkug0HxA== + dependencies: + duplexer "^0.1.1" + minimist "^1.2.0" + through "^2.3.4" + +supports-color@^5.3.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" + integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== + dependencies: + has-flag "^3.0.0" + +supports-color@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-6.1.0.tgz#0764abc69c63d5ac842dd4867e8d025e880df8f3" + integrity sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ== + dependencies: + has-flag "^3.0.0" + +supports-color@^7.0.0, supports-color@^7.1.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da" + integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== + dependencies: + has-flag "^4.0.0" + +supports-color@^8.0.0: + version "8.1.1" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-8.1.1.tgz#cd6fc17e28500cff56c1b86c0a7fd4a54a73005c" + integrity sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q== + dependencies: + has-flag "^4.0.0" + +supports-hyperlinks@^2.0.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/supports-hyperlinks/-/supports-hyperlinks-2.2.0.tgz#4f77b42488765891774b70c79babd87f9bd594bb" + integrity sha512-6sXEzV5+I5j8Bmq9/vUphGRM/RJNT9SCURJLjwfOg51heRtguGWDzcaBlgAzKhQa0EVNpPEKzQuBwZ8S8WaCeQ== + dependencies: + has-flag "^4.0.0" + supports-color "^7.0.0" + +sver-compat@^1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/sver-compat/-/sver-compat-1.5.0.tgz#3cf87dfeb4d07b4a3f14827bc186b3fd0c645cd8" + integrity sha1-PPh9/rTQe0o/FIJ7wYaz/QxkXNg= + dependencies: + es6-iterator "^2.0.1" + es6-symbol "^3.1.1" + +symbol-tree@^3.2.4: + version "3.2.4" + resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2" + integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw== + +table-layout@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/table-layout/-/table-layout-1.0.2.tgz#c4038a1853b0136d63365a734b6931cf4fad4a04" + integrity sha512-qd/R7n5rQTRFi+Zf2sk5XVVd9UQl6ZkduPFC3S7WEGJAmetDTjY3qPN50eSKzwuzEyQKy5TN2TiZdkIjos2L6A== + dependencies: + array-back "^4.0.1" + deep-extend "~0.6.0" + typical "^5.2.0" + wordwrapjs "^4.0.0" + +table@^6.0.9: + version "6.7.1" + resolved "https://registry.yarnpkg.com/table/-/table-6.7.1.tgz#ee05592b7143831a8c94f3cee6aae4c1ccef33e2" + integrity sha512-ZGum47Yi6KOOFDE8m223td53ath2enHcYLgOCjGr5ngu8bdIARQk6mN/wRMv4yMRcHnCSnHbCEha4sobQx5yWg== + dependencies: + ajv "^8.0.1" + lodash.clonedeep "^4.5.0" + lodash.truncate "^4.4.2" + slice-ansi "^4.0.0" + string-width "^4.2.0" + strip-ansi "^6.0.0" + +tar@^4.4.12: + version "4.4.13" + resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.13.tgz#43b364bc52888d555298637b10d60790254ab525" + integrity sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA== + dependencies: + chownr "^1.1.1" + fs-minipass "^1.2.5" + minipass "^2.8.6" + minizlib "^1.2.1" + mkdirp "^0.5.0" + safe-buffer "^5.1.2" + yallist "^3.0.3" + +tar@^6.0.2, tar@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.0.tgz#d1724e9bcc04b977b18d5c573b333a2207229a83" + integrity sha512-DUCttfhsnLCjwoDoFcI+B2iJgYa93vBnDUATYEeRx6sntCTdN01VnqsIuTlALXla/LWooNg0yEGeB+Y8WdFxGA== + dependencies: + chownr "^2.0.0" + fs-minipass "^2.0.0" + minipass "^3.0.0" + minizlib "^2.1.1" + mkdirp "^1.0.3" + yallist "^4.0.0" + +temp-dir@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/temp-dir/-/temp-dir-1.0.0.tgz#0a7c0ea26d3a39afa7e0ebea9c1fc0bc4daa011d" + integrity sha1-CnwOom06Oa+n4OvqnB/AvE2qAR0= + +temp-write@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/temp-write/-/temp-write-4.0.0.tgz#cd2e0825fc826ae72d201dc26eef3bf7e6fc9320" + integrity sha512-HIeWmj77uOOHb0QX7siN3OtwV3CTntquin6TNVg6SHOqCP3hYKmox90eeFOGaY1MqJ9WYDDjkyZrW6qS5AWpbw== + dependencies: + graceful-fs "^4.1.15" + is-stream "^2.0.0" + make-dir "^3.0.0" + temp-dir "^1.0.0" + uuid "^3.3.2" + +terminal-link@^2.0.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/terminal-link/-/terminal-link-2.1.1.tgz#14a64a27ab3c0df933ea546fba55f2d078edc994" + integrity sha512-un0FmiRUQNr5PJqy9kP7c40F5BOfpGlYTrxonDChEZB7pzZxRNp/bt+ymiy9/npwXya9KH99nJ/GXFIiUkYGFQ== + dependencies: + ansi-escapes "^4.2.1" + supports-hyperlinks "^2.0.0" + +test-exclude@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/test-exclude/-/test-exclude-6.0.0.tgz#04a8698661d805ea6fa293b6cb9e63ac044ef15e" + integrity sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w== + dependencies: + "@istanbuljs/schema" "^0.1.2" + glob "^7.1.4" + minimatch "^3.0.4" + +text-extensions@^1.0.0: + version "1.9.0" + resolved "https://registry.yarnpkg.com/text-extensions/-/text-extensions-1.9.0.tgz#1853e45fee39c945ce6f6c36b2d659b5aabc2a26" + integrity sha512-wiBrwC1EhBelW12Zy26JeOUkQ5mRu+5o8rpsJk5+2t+Y5vE7e842qtZDQ2g1NpX/29HdyFeJ4nSIhI47ENSxlQ== + +text-table@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" + integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ= + +throat@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/throat/-/throat-6.0.1.tgz#d514fedad95740c12c2d7fc70ea863eb51ade375" + integrity sha512-8hmiGIJMDlwjg7dlJ4yKGLK8EsYqKgPWbG3b4wjJddKNwc7N7Dpn08Df4szr/sZdMVeOstrdYSsqzX6BYbcB+w== + +through2-filter@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/through2-filter/-/through2-filter-3.0.0.tgz#700e786df2367c2c88cd8aa5be4cf9c1e7831254" + integrity sha512-jaRjI2WxN3W1V8/FMZ9HKIBXixtiqs3SQSX4/YGIiP3gL6djW48VoZq9tDqeCWs3MT8YY5wb/zli8VW8snY1CA== + dependencies: + through2 "~2.0.0" + xtend "~4.0.0" + +through2@^2.0.0, through2@^2.0.3, through2@~2.0.0: + version "2.0.5" + resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd" + integrity sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ== + dependencies: + readable-stream "~2.3.6" + xtend "~4.0.1" + +through2@^3.0.0, through2@^3.0.1: + version "3.0.2" + resolved "https://registry.yarnpkg.com/through2/-/through2-3.0.2.tgz#99f88931cfc761ec7678b41d5d7336b5b6a07bf4" + integrity sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ== + dependencies: + inherits "^2.0.4" + readable-stream "2 || 3" + +through2@^4.0.0: + version "4.0.2" + resolved "https://registry.yarnpkg.com/through2/-/through2-4.0.2.tgz#a7ce3ac2a7a8b0b966c80e7c49f0484c3b239764" + integrity sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw== + dependencies: + readable-stream "3" + +through@2, "through@>=2.2.7 <3", through@^2.3.4, through@^2.3.6: + version "2.3.8" + resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" + integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= + +time-stamp@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/time-stamp/-/time-stamp-1.1.0.tgz#764a5a11af50561921b133f3b44e618687e0f5c3" + integrity sha1-dkpaEa9QVhkhsTPztE5hhofg9cM= + +timers-ext@^0.1.7: + version "0.1.7" + resolved "https://registry.yarnpkg.com/timers-ext/-/timers-ext-0.1.7.tgz#6f57ad8578e07a3fb9f91d9387d65647555e25c6" + integrity sha512-b85NUNzTSdodShTIbky6ZF02e8STtVVfD+fu4aXXShEELpozH+bCpJLYMPZbsABN2wDH7fJpqIoXxJpzbf0NqQ== + dependencies: + es5-ext "~0.10.46" + next-tick "1" + +tmp@^0.0.33: + version "0.0.33" + resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9" + integrity sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw== + dependencies: + os-tmpdir "~1.0.2" + +tmpl@1.0.x: + version "1.0.4" + resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1" + integrity sha1-I2QN17QtAEM5ERQIIOXPRA5SHdE= + +to-absolute-glob@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/to-absolute-glob/-/to-absolute-glob-2.0.2.tgz#1865f43d9e74b0822db9f145b78cff7d0f7c849b" + integrity sha1-GGX0PZ50sIItufFFt4z/fQ98hJs= + dependencies: + is-absolute "^1.0.0" + is-negated-glob "^1.0.0" + +to-fast-properties@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" + integrity sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4= + +to-object-path@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/to-object-path/-/to-object-path-0.3.0.tgz#297588b7b0e7e0ac08e04e672f85c1f4999e17af" + integrity sha1-KXWIt7Dn4KwI4E5nL4XB9JmeF68= + dependencies: + kind-of "^3.0.2" + +to-regex-range@^2.1.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-2.1.1.tgz#7c80c17b9dfebe599e27367e0d4dd5590141db38" + integrity sha1-fIDBe53+vlmeJzZ+DU3VWQFB2zg= + dependencies: + is-number "^3.0.0" + repeat-string "^1.6.1" + +to-regex-range@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4" + integrity sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ== + dependencies: + is-number "^7.0.0" + +to-regex@^3.0.1, to-regex@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/to-regex/-/to-regex-3.0.2.tgz#13cfdd9b336552f30b51f33a8ae1b42a7a7599ce" + integrity sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw== + dependencies: + define-property "^2.0.2" + extend-shallow "^3.0.2" + regex-not "^1.0.2" + safe-regex "^1.1.0" + +to-through@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/to-through/-/to-through-2.0.0.tgz#fc92adaba072647bc0b67d6b03664aa195093af6" + integrity sha1-/JKtq6ByZHvAtn1rA2ZKoZUJOvY= + dependencies: + through2 "^2.0.3" + +tough-cookie@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.0.0.tgz#d822234eeca882f991f0f908824ad2622ddbece4" + integrity sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg== + dependencies: + psl "^1.1.33" + punycode "^2.1.1" + universalify "^0.1.2" + +tough-cookie@~2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" + integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== + dependencies: + psl "^1.1.28" + punycode "^2.1.1" + +tr46@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" + integrity sha512-15Ih7phfcdP5YxqiB+iDtLoaTz4Nd35+IiAv0kQ5FNKHzXgdWqPoTIqEDDJmXceQt4JZk6lVPT8lnDlPpGDppw== + dependencies: + punycode "^2.1.1" + +trim-newlines@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-3.0.1.tgz#260a5d962d8b752425b32f3a7db0dcacd176c144" + integrity sha512-c1PTsA3tYrIsLGkJkzHF+w9F2EyxfXGo4UyJc4pFL++FMjnq0HJS69T3M7d//gKrFKwy429bouPescbjecU+Zw== + +trim-newlines@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-4.0.2.tgz#d6aaaf6a0df1b4b536d183879a6b939489808c7c" + integrity sha512-GJtWyq9InR/2HRiLZgpIKv+ufIKrVrvjQWEj7PxAXNc5dwbNJkqhAUoAGgzRmULAnoOM5EIpveYd3J2VeSAIew== + +trim-off-newlines@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/trim-off-newlines/-/trim-off-newlines-1.0.1.tgz#9f9ba9d9efa8764c387698bcbfeb2c848f11adb3" + integrity sha1-n5up2e+odkw4dpi8v+sshI8RrbM= + +ts-jest@27.0.3: + version "27.0.3" + resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-27.0.3.tgz#808492f022296cde19390bb6ad627c8126bf93f8" + integrity sha512-U5rdMjnYam9Ucw+h0QvtNDbc5+88nxt7tbIvqaZUhFrfG4+SkWhMXjejCLVGcpILTPuV+H3W/GZDZrnZFpPeXw== + dependencies: + bs-logger "0.x" + buffer-from "1.x" + fast-json-stable-stringify "2.x" + jest-util "^27.0.0" + json5 "2.x" + lodash "4.x" + make-error "1.x" + mkdirp "1.x" + semver "7.x" + yargs-parser "20.x" + +ts-node@10.1.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.1.0.tgz#e656d8ad3b61106938a867f69c39a8ba6efc966e" + integrity sha512-6szn3+J9WyG2hE+5W8e0ruZrzyk1uFLYye6IGMBadnOzDh8aP7t8CbFpsfCiEx2+wMixAhjFt7lOZC4+l+WbEA== + dependencies: + "@tsconfig/node10" "^1.0.7" + "@tsconfig/node12" "^1.0.7" + "@tsconfig/node14" "^1.0.0" + "@tsconfig/node16" "^1.0.1" + arg "^4.1.0" + create-require "^1.1.0" + diff "^4.0.1" + make-error "^1.1.1" + source-map-support "^0.5.17" + yn "3.1.1" + +tslib@^1.8.1, tslib@^1.9.0: + version "1.14.1" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" + integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== + +tslib@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.0.tgz#803b8cdab3e12ba581a4ca41c8839bbb0dacb09e" + integrity sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg== + +tslib@~2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.1.0.tgz#da60860f1c2ecaa5703ab7d39bc05b6bf988b97a" + integrity sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A== + +tsutils@^3.21.0: + version "3.21.0" + resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623" + integrity sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA== + dependencies: + tslib "^1.8.1" + +tunnel-agent@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" + integrity sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0= + dependencies: + safe-buffer "^5.0.1" + +tweetnacl@^0.14.3, tweetnacl@~0.14.0: + version "0.14.5" + resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" + integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q= + +type-check@^0.4.0, type-check@~0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" + integrity sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew== + dependencies: + prelude-ls "^1.2.1" + +type-check@~0.3.2: + version "0.3.2" + resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72" + integrity sha1-WITKtRLPHTVeP7eE8wgEsrUg23I= + dependencies: + prelude-ls "~1.1.2" + +type-detect@4.0.8: + version "4.0.8" + resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c" + integrity sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g== + +type-fest@^0.13.1: + version "0.13.1" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.13.1.tgz#0172cb5bce80b0bd542ea348db50c7e21834d934" + integrity sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg== + +type-fest@^0.18.0: + version "0.18.1" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.18.1.tgz#db4bc151a4a2cf4eebf9add5db75508db6cc841f" + integrity sha512-OIAYXk8+ISY+qTOwkHtKqzAuxchoMiD9Udx+FSGQDuiRR+PJKJHc2NJAXlbhkGwTt/4/nKZxELY1w3ReWOL8mw== + +type-fest@^0.20.2: + version "0.20.2" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.20.2.tgz#1bf207f4b28f91583666cb5fbd327887301cd5f4" + integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ== + +type-fest@^0.21.3: + version "0.21.3" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.21.3.tgz#d260a24b0198436e133fa26a524a6d65fa3b2e37" + integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w== + +type-fest@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.4.1.tgz#8bdf77743385d8a4f13ba95f610f5ccd68c728f8" + integrity sha512-IwzA/LSfD2vC1/YDYMv/zHP4rDF1usCwllsDpbolT3D4fUepIO7f9K70jjmUewU/LmGUKJcwcVtDCpnKk4BPMw== + +type-fest@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.6.0.tgz#8d2a2370d3df886eb5c90ada1c5bf6188acf838b" + integrity sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg== + +type-fest@^0.8.1: + version "0.8.1" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.8.1.tgz#09e249ebde851d3b1e48d27c105444667f17b83d" + integrity sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA== + +type-fest@^1.0.1, type-fest@^1.2.1, type-fest@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-1.2.2.tgz#1930bc36b2064f7ab4aa307a6d1b65965199c698" + integrity sha512-pfkPYCcuV0TJoo/jlsUeWNV8rk7uMU6ocnYNvca1Vu+pyKi8Rl8Zo2scPt9O72gCsXIm+dMxOOWuA3VFDSdzWA== + +type@^1.0.1: + version "1.2.0" + resolved "https://registry.yarnpkg.com/type/-/type-1.2.0.tgz#848dd7698dafa3e54a6c479e759c4bc3f18847a0" + integrity sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg== + +type@^2.0.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/type/-/type-2.5.0.tgz#0a2e78c2e77907b252abe5f298c1b01c63f0db3d" + integrity sha512-180WMDQaIMm3+7hGXWf12GtdniDEy7nYcyFMKJn/eZz/6tSLXrUN9V0wKSbMjej0I1WHWbpREDEKHtqPQa9NNw== + +typedarray-to-buffer@^3.1.5: + version "3.1.5" + resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080" + integrity sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q== + dependencies: + is-typedarray "^1.0.0" + +typedarray@^0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" + integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= + +typedoc-default-themes@^0.12.10: + version "0.12.10" + resolved "https://registry.yarnpkg.com/typedoc-default-themes/-/typedoc-default-themes-0.12.10.tgz#614c4222fe642657f37693ea62cad4dafeddf843" + integrity sha512-fIS001cAYHkyQPidWXmHuhs8usjP5XVJjWB8oZGqkTowZaz3v7g3KDZeeqE82FBrmkAnIBOY3jgy7lnPnqATbA== + +typedoc@0.21.4: + version "0.21.4" + resolved "https://registry.yarnpkg.com/typedoc/-/typedoc-0.21.4.tgz#fced3cffdc30180db60a5dbfec9dbbb273cb5b31" + integrity sha512-slZQhvD9U0d9KacktYAyuNMMOXJRFNHy+Gd8xY2Qrqq3eTTTv3frv3N4au/cFnab9t3T5WA0Orb6QUjMc+1bDA== + dependencies: + glob "^7.1.7" + handlebars "^4.7.7" + lunr "^2.3.9" + marked "^2.1.1" + minimatch "^3.0.0" + progress "^2.0.3" + shiki "^0.9.3" + typedoc-default-themes "^0.12.10" + +typescript@4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.0.2.tgz#7ea7c88777c723c681e33bf7988be5d008d05ac2" + integrity sha512-e4ERvRV2wb+rRZ/IQeb3jm2VxBsirQLpQhdxplZ2MEzGvDkkMmPglecnNDfSUBivMjP93vRbngYYDQqQ/78bcQ== + +typical@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/typical/-/typical-4.0.0.tgz#cbeaff3b9d7ae1e2bbfaf5a4e6f11eccfde94fc4" + integrity sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw== + +typical@^5.2.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/typical/-/typical-5.2.0.tgz#4daaac4f2b5315460804f0acf6cb69c52bb93066" + integrity sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg== + +uglify-js@^3.1.4: + version "3.13.10" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.13.10.tgz#a6bd0d28d38f592c3adb6b180ea6e07e1e540a8d" + integrity sha512-57H3ACYFXeo1IaZ1w02sfA71wI60MGco/IQFjOqK+WtKoprh7Go2/yvd2HPtoJILO2Or84ncLccI4xoHMTSbGg== + +uid-number@0.0.6: + version "0.0.6" + resolved "https://registry.yarnpkg.com/uid-number/-/uid-number-0.0.6.tgz#0ea10e8035e8eb5b8e4449f06da1c730663baa81" + integrity sha1-DqEOgDXo61uOREnwbaHHMGY7qoE= + +umask@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/umask/-/umask-1.1.0.tgz#f29cebf01df517912bb58ff9c4e50fde8e33320d" + integrity sha1-8pzr8B31F5ErtY/5xOUP3o4zMg0= + +unbox-primitive@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.1.tgz#085e215625ec3162574dc8859abee78a59b14471" + integrity sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw== + dependencies: + function-bind "^1.1.1" + has-bigints "^1.0.1" + has-symbols "^1.0.2" + which-boxed-primitive "^1.0.2" + +unc-path-regex@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/unc-path-regex/-/unc-path-regex-0.1.2.tgz#e73dd3d7b0d7c5ed86fbac6b0ae7d8c6a69d50fa" + integrity sha1-5z3T17DXxe2G+6xrCufYxqadUPo= + +undertaker-registry@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/undertaker-registry/-/undertaker-registry-1.0.1.tgz#5e4bda308e4a8a2ae584f9b9a4359a499825cc50" + integrity sha1-XkvaMI5KiirlhPm5pDWaSZglzFA= + +undertaker@^1.2.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/undertaker/-/undertaker-1.3.0.tgz#363a6e541f27954d5791d6fa3c1d321666f86d18" + integrity sha512-/RXwi5m/Mu3H6IHQGww3GNt1PNXlbeCuclF2QYR14L/2CHPz3DFZkvB5hZ0N/QUkiXWCACML2jXViIQEQc2MLg== + dependencies: + arr-flatten "^1.0.1" + arr-map "^2.0.0" + bach "^1.0.0" + collection-map "^1.0.0" + es6-weak-map "^2.0.1" + fast-levenshtein "^1.0.0" + last-run "^1.1.0" + object.defaults "^1.0.0" + object.reduce "^1.0.0" + undertaker-registry "^1.0.0" + +union-value@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/union-value/-/union-value-1.0.1.tgz#0b6fe7b835aecda61c6ea4d4f02c14221e109847" + integrity sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg== + dependencies: + arr-union "^3.1.0" + get-value "^2.0.6" + is-extendable "^0.1.1" + set-value "^2.0.1" + +unique-filename@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/unique-filename/-/unique-filename-1.1.1.tgz#1d69769369ada0583103a1e6ae87681b56573230" + integrity sha512-Vmp0jIp2ln35UTXuryvjzkjGdRyf9b2lTXuSYUiPmzRcl3FDtYqAwOnTJkAngD9SWhnoJzDbTKwaOrZ+STtxNQ== + dependencies: + unique-slug "^2.0.0" + +unique-slug@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/unique-slug/-/unique-slug-2.0.2.tgz#baabce91083fc64e945b0f3ad613e264f7cd4e6c" + integrity sha512-zoWr9ObaxALD3DOPfjPSqxt4fnZiWblxHIgeWqW8x7UqDzEtHEQLzji2cuJYQFCU6KmoJikOYAZlrTHHebjx2w== + dependencies: + imurmurhash "^0.1.4" + +unique-stream@^2.0.2: + version "2.3.1" + resolved "https://registry.yarnpkg.com/unique-stream/-/unique-stream-2.3.1.tgz#c65d110e9a4adf9a6c5948b28053d9a8d04cbeac" + integrity sha512-2nY4TnBE70yoxHkDli7DMazpWiP7xMdCYqU2nBRO0UB+ZpEkGsSija7MvmvnZFUeC+mrgiUfcHSr3LmRFIg4+A== + dependencies: + json-stable-stringify-without-jsonify "^1.0.1" + through2-filter "^3.0.0" + +universal-user-agent@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/universal-user-agent/-/universal-user-agent-6.0.0.tgz#3381f8503b251c0d9cd21bc1de939ec9df5480ee" + integrity sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w== + +universalify@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" + integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== + +universalify@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.0.tgz#75a4984efedc4b08975c5aeb73f530d02df25717" + integrity sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ== + +unset-value@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-1.0.0.tgz#8376873f7d2335179ffb1e6fc3a8ed0dfc8ab559" + integrity sha1-g3aHP30jNRef+x5vw6jtDfyKtVk= + dependencies: + has-value "^0.3.1" + isobject "^3.0.0" + +upath@^1.1.1: + version "1.2.0" + resolved "https://registry.yarnpkg.com/upath/-/upath-1.2.0.tgz#8f66dbcd55a883acdae4408af8b035a5044c1894" + integrity sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg== + +upath@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/upath/-/upath-2.0.1.tgz#50c73dea68d6f6b990f51d279ce6081665d61a8b" + integrity sha512-1uEe95xksV1O0CYKXo8vQvN1JEbtJp7lb7C5U9HMsIp6IVwntkH/oNUzyVNQSd4S1sYk2FpSSW44FqMc8qee5w== + +uri-js@^4.2.2: + version "4.4.1" + resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" + integrity sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg== + dependencies: + punycode "^2.1.0" + +urix@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72" + integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI= + +use@^3.1.0: + version "3.1.1" + resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f" + integrity sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ== + +util-deprecate@^1.0.1, util-deprecate@~1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" + integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= + +util-promisify@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/util-promisify/-/util-promisify-2.1.0.tgz#3c2236476c4d32c5ff3c47002add7c13b9a82a53" + integrity sha1-PCI2R2xNMsX/PEcAKt18E7moKlM= + dependencies: + object.getownpropertydescriptors "^2.0.3" + +uuid@^3.3.2: + version "3.4.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" + integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== + +v8-compile-cache@^2.0.3: + version "2.3.0" + resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" + integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA== + +v8-to-istanbul@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-8.0.0.tgz#4229f2a99e367f3f018fa1d5c2b8ec684667c69c" + integrity sha512-LkmXi8UUNxnCC+JlH7/fsfsKr5AU110l+SYGJimWNkWhxbN5EyeOtm1MJ0hhvqMMOhGwBj1Fp70Yv9i+hX0QAg== + dependencies: + "@types/istanbul-lib-coverage" "^2.0.1" + convert-source-map "^1.6.0" + source-map "^0.7.3" + +v8flags@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/v8flags/-/v8flags-3.2.0.tgz#b243e3b4dfd731fa774e7492128109a0fe66d656" + integrity sha512-mH8etigqMfiGWdeXpaaqGfs6BndypxusHHcv2qSHyZkGEznCd/qAXCWWRzeowtL54147cktFOC4P5y+kl8d8Jg== + dependencies: + homedir-polyfill "^1.0.1" + +validate-npm-package-license@^3.0.1, validate-npm-package-license@^3.0.4: + version "3.0.4" + resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz#fc91f6b9c7ba15c857f4cb2c5defeec39d4f410a" + integrity sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew== + dependencies: + spdx-correct "^3.0.0" + spdx-expression-parse "^3.0.0" + +validate-npm-package-name@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/validate-npm-package-name/-/validate-npm-package-name-3.0.0.tgz#5fa912d81eb7d0c74afc140de7317f0ca7df437e" + integrity sha1-X6kS2B630MdK/BQN5zF/DKffQ34= + dependencies: + builtins "^1.0.3" + +value-or-function@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/value-or-function/-/value-or-function-3.0.0.tgz#1c243a50b595c1be54a754bfece8563b9ff8d813" + integrity sha1-HCQ6ULWVwb5Up1S/7OhWO5/42BM= + +verror@1.10.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" + integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA= + dependencies: + assert-plus "^1.0.0" + core-util-is "1.0.2" + extsprintf "^1.2.0" + +vinyl-fs@^3.0.0, vinyl-fs@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/vinyl-fs/-/vinyl-fs-3.0.3.tgz#c85849405f67428feabbbd5c5dbdd64f47d31bc7" + integrity sha512-vIu34EkyNyJxmP0jscNzWBSygh7VWhqun6RmqVfXePrOwi9lhvRs//dOaGOTRUQr4tx7/zd26Tk5WeSVZitgng== + dependencies: + fs-mkdirp-stream "^1.0.0" + glob-stream "^6.1.0" + graceful-fs "^4.0.0" + is-valid-glob "^1.0.0" + lazystream "^1.0.0" + lead "^1.0.0" + object.assign "^4.0.4" + pumpify "^1.3.5" + readable-stream "^2.3.3" + remove-bom-buffer "^3.0.0" + remove-bom-stream "^1.2.0" + resolve-options "^1.1.0" + through2 "^2.0.0" + to-through "^2.0.0" + value-or-function "^3.0.0" + vinyl "^2.0.0" + vinyl-sourcemap "^1.1.0" + +vinyl-sourcemap@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/vinyl-sourcemap/-/vinyl-sourcemap-1.1.0.tgz#92a800593a38703a8cdb11d8b300ad4be63b3e16" + integrity sha1-kqgAWTo4cDqM2xHYswCtS+Y7PhY= + dependencies: + append-buffer "^1.0.2" + convert-source-map "^1.5.0" + graceful-fs "^4.1.6" + normalize-path "^2.1.1" + now-and-later "^2.0.0" + remove-bom-buffer "^3.0.0" + vinyl "^2.0.0" + +vinyl-sourcemaps-apply@^0.2.0: + version "0.2.1" + resolved "https://registry.yarnpkg.com/vinyl-sourcemaps-apply/-/vinyl-sourcemaps-apply-0.2.1.tgz#ab6549d61d172c2b1b87be5c508d239c8ef87705" + integrity sha1-q2VJ1h0XLCsbh75cUI0jnI74dwU= + dependencies: + source-map "^0.5.1" + +vinyl@2.x, vinyl@^2.0.0, vinyl@^2.1.0: + version "2.2.1" + resolved "https://registry.yarnpkg.com/vinyl/-/vinyl-2.2.1.tgz#23cfb8bbab5ece3803aa2c0a1eb28af7cbba1974" + integrity sha512-LII3bXRFBZLlezoG5FfZVcXflZgWP/4dCwKtxd5ky9+LOtM4CS3bIRQsmR1KMnMW07jpE8fqR2lcxPZ+8sJIcw== + dependencies: + clone "^2.1.1" + clone-buffer "^1.0.0" + clone-stats "^1.0.0" + cloneable-readable "^1.0.0" + remove-trailing-separator "^1.0.1" + replace-ext "^1.0.0" + +vscode-textmate@5.2.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.2.0.tgz#01f01760a391e8222fe4f33fbccbd1ad71aed74e" + integrity sha512-Uw5ooOQxRASHgu6C7GVvUxisKXfSgW4oFlO+aa+PAkgmH89O3CXxEEzNRNtHSqtXFTl0nAC1uYj0GMSH27uwtQ== + +w3c-hr-time@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz#0a89cdf5cc15822df9c360543676963e0cc308cd" + integrity sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ== + dependencies: + browser-process-hrtime "^1.0.0" + +w3c-xmlserializer@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/w3c-xmlserializer/-/w3c-xmlserializer-2.0.0.tgz#3e7104a05b75146cc60f564380b7f683acf1020a" + integrity sha512-4tzD0mF8iSiMiNs30BiLO3EpfGLZUT2MSX/G+o7ZywDzliWQ3OPtTZ0PTC3B3ca1UAf4cJMHB+2Bf56EriJuRA== + dependencies: + xml-name-validator "^3.0.0" + +walker@^1.0.7: + version "1.0.7" + resolved "https://registry.yarnpkg.com/walker/-/walker-1.0.7.tgz#2f7f9b8fd10d677262b18a884e28d19618e028fb" + integrity sha1-L3+bj9ENZ3JisYqITijRlhjgKPs= + dependencies: + makeerror "1.0.x" + +wcwidth@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/wcwidth/-/wcwidth-1.0.1.tgz#f0b0dcf915bc5ff1528afadb2c0e17b532da2fe8" + integrity sha1-8LDc+RW8X/FSivrbLA4XtTLaL+g= + dependencies: + defaults "^1.0.3" + +web-streams-polyfill@3.0.3, web-streams-polyfill@~3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.0.3.tgz#f49e487eedeca47a207c1aee41ee5578f884b42f" + integrity sha512-d2H/t0eqRNM4w2WvmTdoeIvzAUSpK7JmATB8Nr2lb7nQ9BTIJVjbQ/TRFVEh2gUH1HwclPdoPtfMoFfetXaZnA== + +webidl-conversions@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-5.0.0.tgz#ae59c8a00b121543a2acc65c0434f57b0fc11aff" + integrity sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA== + +webidl-conversions@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-6.1.0.tgz#9111b4d7ea80acd40f5270d666621afa78b69514" + integrity sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w== + +whatwg-encoding@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz#5abacf777c32166a51d085d6b4f3e7d27113ddb0" + integrity sha512-b5lim54JOPN9HtzvK9HFXvBma/rnfFeqsic0hSpjtDbVxR3dJKLc+KB4V6GgiGOvl7CY/KNh8rxSo9DKQrnUEw== + dependencies: + iconv-lite "0.4.24" + +whatwg-mimetype@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf" + integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g== + +whatwg-url@^8.0.0, whatwg-url@^8.4.0, whatwg-url@^8.5.0: + version "8.7.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-8.7.0.tgz#656a78e510ff8f3937bc0bcbe9f5c0ac35941b77" + integrity sha512-gAojqb/m9Q8a5IV96E3fHJM70AzCkgt4uXYX2O7EmuyOnLrViCQlsEBmF9UQIu3/aeAIp2U17rtbpZWNntQqdg== + dependencies: + lodash "^4.7.0" + tr46 "^2.1.0" + webidl-conversions "^6.1.0" + +which-boxed-primitive@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6" + integrity sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg== + dependencies: + is-bigint "^1.0.1" + is-boolean-object "^1.1.0" + is-number-object "^1.0.4" + is-string "^1.0.5" + is-symbol "^1.0.3" + +which-module@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/which-module/-/which-module-1.0.0.tgz#bba63ca861948994ff307736089e3b96026c2a4f" + integrity sha1-u6Y8qGGUiZT/MHc2CJ47lgJsKk8= + +which@^1.2.14, which@^1.2.9, which@^1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" + integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ== + dependencies: + isexe "^2.0.0" + +which@^2.0.1, which@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" + integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA== + dependencies: + isexe "^2.0.0" + +wide-align@^1.1.0: + version "1.1.3" + resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.3.tgz#ae074e6bdc0c14a431e804e624549c633b000457" + integrity sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA== + dependencies: + string-width "^1.0.2 || 2" + +word-wrap@^1.2.3, word-wrap@~1.2.3: + version "1.2.3" + resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" + integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== + +wordwrap@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" + integrity sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus= + +wordwrapjs@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/wordwrapjs/-/wordwrapjs-4.0.1.tgz#d9790bccfb110a0fc7836b5ebce0937b37a8b98f" + integrity sha512-kKlNACbvHrkpIw6oPeYDSmdCTu2hdMHoyXLTcUKala++lx5Y+wjJ/e474Jqv5abnVmwxw08DiTuHmw69lJGksA== + dependencies: + reduce-flatten "^2.0.0" + typical "^5.2.0" + +wrap-ansi@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-2.1.0.tgz#d8fc3d284dd05794fe84973caecdd1cf824fdd85" + integrity sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU= + dependencies: + string-width "^1.0.1" + strip-ansi "^3.0.1" + +wrap-ansi@^6.2.0: + version "6.2.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-6.2.0.tgz#e9393ba07102e6c91a3b221478f0257cd2856e53" + integrity sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + +wrappy@1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" + integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= + +write-file-atomic@^2.4.2: + version "2.4.3" + resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-2.4.3.tgz#1fd2e9ae1df3e75b8d8c367443c692d4ca81f481" + integrity sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ== + dependencies: + graceful-fs "^4.1.11" + imurmurhash "^0.1.4" + signal-exit "^3.0.2" + +write-file-atomic@^3.0.0, write-file-atomic@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-3.0.3.tgz#56bd5c5a5c70481cd19c571bd39ab965a5de56e8" + integrity sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q== + dependencies: + imurmurhash "^0.1.4" + is-typedarray "^1.0.0" + signal-exit "^3.0.2" + typedarray-to-buffer "^3.1.5" + +write-json-file@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/write-json-file/-/write-json-file-3.2.0.tgz#65bbdc9ecd8a1458e15952770ccbadfcff5fe62a" + integrity sha512-3xZqT7Byc2uORAatYiP3DHUUAVEkNOswEWNs9H5KXiicRTvzYzYqKjYc4G7p+8pltvAw641lVByKVtMpf+4sYQ== + dependencies: + detect-indent "^5.0.0" + graceful-fs "^4.1.15" + make-dir "^2.1.0" + pify "^4.0.1" + sort-keys "^2.0.0" + write-file-atomic "^2.4.2" + +write-json-file@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/write-json-file/-/write-json-file-4.3.0.tgz#908493d6fd23225344af324016e4ca8f702dd12d" + integrity sha512-PxiShnxf0IlnQuMYOPPhPkhExoCQuTUNPOa/2JWCYTmBquU9njyyDuwRKN26IZBlp4yn1nt+Agh2HOOBl+55HQ== + dependencies: + detect-indent "^6.0.0" + graceful-fs "^4.1.15" + is-plain-obj "^2.0.0" + make-dir "^3.0.0" + sort-keys "^4.0.0" + write-file-atomic "^3.0.0" + +write-pkg@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/write-pkg/-/write-pkg-4.0.0.tgz#675cc04ef6c11faacbbc7771b24c0abbf2a20039" + integrity sha512-v2UQ+50TNf2rNHJ8NyWttfm/EJUBWMJcx6ZTYZr6Qp52uuegWw/lBkCtCbnYZEmPRNL61m+u67dAmGxo+HTULA== + dependencies: + sort-keys "^2.0.0" + type-fest "^0.4.1" + write-json-file "^3.2.0" + +ws@^7.4.5: + version "7.5.3" + resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.3.tgz#160835b63c7d97bfab418fc1b8a9fced2ac01a74" + integrity sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg== + +xml-name-validator@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" + integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== + +xml2js@0.4.23: + version "0.4.23" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" + integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + +xmlbuilder@~11.0.0: + version "11.0.1" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" + integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== + +xmlchars@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" + integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw== + +xtend@~4.0.0, xtend@~4.0.1: + version "4.0.2" + resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" + integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== + +y18n@^3.2.1: + version "3.2.2" + resolved "https://registry.yarnpkg.com/y18n/-/y18n-3.2.2.tgz#85c901bd6470ce71fc4bb723ad209b70f7f28696" + integrity sha512-uGZHXkHnhF0XeeAPgnKfPv1bgKAYyVvmNL1xlKsPYZPaIHxGti2hHqvOCQv71XMsLxu1QjergkqogUnms5D3YQ== + +y18n@^5.0.5: + version "5.0.8" + resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" + integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== + +yallist@^3.0.0, yallist@^3.0.2, yallist@^3.0.3: + version "3.1.1" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" + integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== + +yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== + +yaml@^1.10.0: + version "1.10.2" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" + integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== + +yargs-parser@20.2.4: + version "20.2.4" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.4.tgz#b42890f14566796f85ae8e3a25290d205f154a54" + integrity sha512-WOkpgNhPTlE73h4VFAFsOnomJVaovO8VqLDzy5saChRBFQFBoMYirowyW+Q9HB4HFF4Z7VZTiG3iSzJJA29yRA== + +yargs-parser@20.x, yargs-parser@^20.2.2, yargs-parser@^20.2.3, yargs-parser@^20.2.9: + version "20.2.9" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee" + integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w== + +yargs-parser@^18.1.3: + version "18.1.3" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-18.1.3.tgz#be68c4975c6b2abf469236b0c870362fab09a7b0" + integrity sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ== + dependencies: + camelcase "^5.0.0" + decamelize "^1.2.0" + +yargs-parser@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-5.0.1.tgz#7ede329c1d8cdbbe209bd25cdb990e9b1ebbb394" + integrity sha512-wpav5XYiddjXxirPoCTUPbqM0PXvJ9hiBMvuJgInvo4/lAOTZzUprArw17q2O1P2+GHhbBr18/iQwjL5Z9BqfA== + dependencies: + camelcase "^3.0.0" + object.assign "^4.1.0" + +yargs@^16.0.3, yargs@^16.2.0: + version "16.2.0" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-16.2.0.tgz#1c82bf0f6b6a66eafce7ef30e376f49a12477f66" + integrity sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw== + dependencies: + cliui "^7.0.2" + escalade "^3.1.1" + get-caller-file "^2.0.5" + require-directory "^2.1.1" + string-width "^4.2.0" + y18n "^5.0.5" + yargs-parser "^20.2.2" + +yargs@^7.1.0: + version "7.1.2" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-7.1.2.tgz#63a0a5d42143879fdbb30370741374e0641d55db" + integrity sha512-ZEjj/dQYQy0Zx0lgLMLR8QuaqTihnxirir7EwUHp1Axq4e3+k8jXU5K0VLbNvedv1f4EWtBonDIZm0NUr+jCcA== + dependencies: + camelcase "^3.0.0" + cliui "^3.2.0" + decamelize "^1.1.1" + get-caller-file "^1.0.1" + os-locale "^1.4.0" + read-pkg-up "^1.0.1" + require-directory "^2.1.1" + require-main-filename "^1.0.1" + set-blocking "^2.0.0" + string-width "^1.0.2" + which-module "^1.0.0" + y18n "^3.2.1" + yargs-parser "^5.0.1" + +yn@3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50" + integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q== + +yocto-queue@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" + integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== |