diff options
Diffstat (limited to 'src/arrow/js/gulp')
| -rw-r--r-- | src/arrow/js/gulp/argv.js | 39 | ||||
| -rw-r--r-- | src/arrow/js/gulp/arrow-task.js | 70 | ||||
| -rw-r--r-- | src/arrow/js/gulp/clean-task.js | 34 | ||||
| -rw-r--r-- | src/arrow/js/gulp/closure-task.js | 213 | ||||
| -rw-r--r-- | src/arrow/js/gulp/compile-task.js | 35 | ||||
| -rw-r--r-- | src/arrow/js/gulp/memoize-task.js | 38 | ||||
| -rw-r--r-- | src/arrow/js/gulp/package-task.js | 121 | ||||
| -rw-r--r-- | src/arrow/js/gulp/test-task.js | 186 | ||||
| -rw-r--r-- | src/arrow/js/gulp/typescript-task.js | 78 | ||||
| -rw-r--r-- | src/arrow/js/gulp/util.js | 200 |
10 files changed, 1014 insertions, 0 deletions
diff --git a/src/arrow/js/gulp/argv.js b/src/arrow/js/gulp/argv.js new file mode 100644 index 000000000..0acdad7d5 --- /dev/null +++ b/src/arrow/js/gulp/argv.js @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const argv = require(`command-line-args`)([ + { name: `all`, type: Boolean }, + { name: 'verbose', alias: `v`, type: Boolean }, + { name: `target`, type: String, defaultValue: `` }, + { name: `module`, type: String, defaultValue: `` }, + { name: `coverage`, type: Boolean, defaultValue: false }, + { name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] }, + { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] }, +], { partial: true }); + +const { targets, modules } = argv; + +if (argv.target === `src`) { + argv.target && !targets.length && targets.push(argv.target); +} else { + argv.target && !targets.length && targets.push(argv.target); + argv.module && !modules.length && modules.push(argv.module); + (argv.all || !targets.length) && targets.push(`all`); + (argv.all || !modules.length) && modules.push(`all`); +} + +module.exports = { argv, targets, modules }; diff --git a/src/arrow/js/gulp/arrow-task.js b/src/arrow/js/gulp/arrow-task.js new file mode 100644 index 000000000..fc85dd72e --- /dev/null +++ b/src/arrow/js/gulp/arrow-task.js @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { + targetDir, observableFromStreams +} = require('./util'); + +const del = require('del'); +const gulp = require('gulp'); +const mkdirp = require('mkdirp'); +const gulpRename = require(`gulp-rename`); +const { memoizeTask } = require('./memoize-task'); +const { + ReplaySubject, + forkJoin: ObservableForkJoin, +} = require('rxjs'); +const { + share +} = require('rxjs/operators'); +const pipeline = require('util').promisify(require('stream').pipeline); + +const arrowTask = ((cache) => memoizeTask(cache, function copyMain(target) { + const out = targetDir(target); + const dtsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.ts`; + const cjsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.js`; + const esmGlob = `${targetDir(`es2015`, `esm`)}/**/*.js`; + const es2015UmdGlob = `${targetDir(`es2015`, `umd`)}/*.js`; + const esnextUmdGlob = `${targetDir(`esnext`, `umd`)}/*.js`; + const cjsSourceMapsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.map`; + const esmSourceMapsGlob = `${targetDir(`es2015`, `esm`)}/**/*.map`; + const es2015UmdSourceMapsGlob = `${targetDir(`es2015`, `umd`)}/*.map`; + const esnextUmdSourceMapsGlob = `${targetDir(`esnext`, `umd`)}/*.map`; + return ObservableForkJoin([ + observableFromStreams(gulp.src(dtsGlob), gulp.dest(out)), // copy d.ts files + observableFromStreams(gulp.src(cjsGlob), gulp.dest(out)), // copy es2015 cjs files + observableFromStreams(gulp.src(cjsSourceMapsGlob), gulp.dest(out)), // copy es2015 cjs sourcemaps + observableFromStreams(gulp.src(esmSourceMapsGlob), gulp.dest(out)), // copy es2015 esm sourcemaps + observableFromStreams(gulp.src(es2015UmdSourceMapsGlob), gulp.dest(out)), // copy es2015 umd sourcemap files, but don't rename + observableFromStreams(gulp.src(esnextUmdSourceMapsGlob), gulp.dest(out)), // copy esnext umd sourcemap files, but don't rename + observableFromStreams(gulp.src(esmGlob), gulpRename((p) => { p.extname = '.mjs'; }), gulp.dest(out)), // copy es2015 esm files and rename to `.mjs` + observableFromStreams(gulp.src(es2015UmdGlob), gulpRename((p) => { p.basename += `.es2015.min`; }), gulp.dest(out)), // copy es2015 umd files and add `.es2015.min` + observableFromStreams(gulp.src(esnextUmdGlob), gulpRename((p) => { p.basename += `.esnext.min`; }), gulp.dest(out)), // copy esnext umd files and add `.esnext.min` + ]).pipe(share({ connector: () => new ReplaySubject(), resetOnError: false, resetOnComplete: false, resetOnRefCountZero: false })); +}))({}); + +const arrowTSTask = ((cache) => memoizeTask(cache, async function copyTS(target, format) { + const out = targetDir(target, format); + await mkdirp(out); + await pipeline(gulp.src(`src/**/*`), gulp.dest(out)); + await del(`${out}/**/*.js`); +}))({}); + + +module.exports = arrowTask; +module.exports.arrowTask = arrowTask; +module.exports.arrowTSTask = arrowTSTask; diff --git a/src/arrow/js/gulp/clean-task.js b/src/arrow/js/gulp/clean-task.js new file mode 100644 index 000000000..0034f9a09 --- /dev/null +++ b/src/arrow/js/gulp/clean-task.js @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const del = require('del'); +const { targetDir } = require('./util'); +const memoizeTask = require('./memoize-task'); +const { catchError } = require('rxjs/operators'); +const { + from: ObservableFrom, + EMPTY: ObservableEmpty, +} = require('rxjs'); + +const cleanTask = ((cache) => memoizeTask(cache, function clean(target, format) { + const dir = targetDir(target, format); + return ObservableFrom(del(dir)) + .pipe(catchError((e) => ObservableEmpty())); +}))({}); + +module.exports = cleanTask; +module.exports.cleanTask = cleanTask;
\ No newline at end of file diff --git a/src/arrow/js/gulp/closure-task.js b/src/arrow/js/gulp/closure-task.js new file mode 100644 index 000000000..6e5a61d82 --- /dev/null +++ b/src/arrow/js/gulp/closure-task.js @@ -0,0 +1,213 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { + targetDir, + mainExport, + esmRequire, + gCCLanguageNames, + publicModulePaths, + observableFromStreams, + shouldRunInChildProcess, + spawnGulpCommandInChildProcess, +} = require('./util'); + +const fs = require('fs'); +const gulp = require('gulp'); +const path = require('path'); +const mkdirp = require('mkdirp'); +const sourcemaps = require('gulp-sourcemaps'); +const { memoizeTask } = require('./memoize-task'); +const { compileBinFiles } = require('./typescript-task'); +const closureCompiler = require('google-closure-compiler').gulp(); + +const closureTask = ((cache) => memoizeTask(cache, async function closure(target, format) { + + if (shouldRunInChildProcess(target, format)) { + return spawnGulpCommandInChildProcess('compile', target, format); + } + + const src = targetDir(target, `cls`); + const srcAbsolute = path.resolve(src); + const out = targetDir(target, format); + const externs = path.join(`${out}/${mainExport}.externs.js`); + const entry_point = path.join(`${src}/${mainExport}.dom.cls.js`); + + const exportedImports = publicModulePaths(srcAbsolute).reduce((entries, publicModulePath) => [ + ...entries, { + publicModulePath, + exports_: getPublicExportedNames(esmRequire(publicModulePath)) + } + ], []); + + await mkdirp(out); + + await Promise.all([ + fs.promises.writeFile(externs, generateExternsFile(exportedImports)), + fs.promises.writeFile(entry_point, generateUMDExportAssignment(srcAbsolute, exportedImports)) + ]); + + return await Promise.all([ + runClosureCompileAsObservable().toPromise(), + compileBinFiles(target, format).toPromise() + ]); + + function runClosureCompileAsObservable() { + return observableFromStreams( + gulp.src([ + /* external libs first */ + `node_modules/flatbuffers/package.json`, + `node_modules/flatbuffers/js/flatbuffers.mjs`, + `${src}/**/*.js` /* <-- then source globs */ + ], { base: `./` }), + sourcemaps.init(), + closureCompiler(createClosureArgs(entry_point, externs, target), { + platform: ['native', 'java', 'javascript'] + }), + // rename the sourcemaps from *.js.map files to *.min.js.map + sourcemaps.write(`.`, { mapFile: (mapPath) => mapPath.replace(`.js.map`, `.${target}.min.js.map`) }), + gulp.dest(out) + ); + } +}))({}); + +module.exports = closureTask; +module.exports.closureTask = closureTask; + +const createClosureArgs = (entry_point, externs, target) => ({ + externs, + entry_point, + third_party: true, + warning_level: `QUIET`, + dependency_mode: `PRUNE`, + rewrite_polyfills: false, + module_resolution: `NODE`, + // formatting: `PRETTY_PRINT`, + // debug: true, + compilation_level: `ADVANCED`, + package_json_entry_names: `module,jsnext:main,main`, + assume_function_wrapper: true, + js_output_file: `${mainExport}.js`, + language_in: gCCLanguageNames[`esnext`], + language_out: gCCLanguageNames[target], + output_wrapper:`${apacheHeader()} +(function (global, factory) { + typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) : + typeof define === 'function' && define.amd ? define(['exports'], factory) : + (factory(global.Arrow = global.Arrow || {})); +}(this, (function (exports) {%output%}.bind(this))));` +}); + +function generateUMDExportAssignment(src, exportedImports) { + return [ + ...exportedImports.map(({ publicModulePath }, i) => { + const p = publicModulePath.slice(src.length + 1); + return (`import * as exports${i} from './${p}';`); + }).filter(Boolean), + 'Object.assign(arguments[0], exports0);' + ].join('\n'); +} + +function generateExternsFile(exportedImports) { + return [ + externsHeader(), + ...exportedImports.reduce((externBodies, { exports_ }) => [ + ...externBodies, ...exports_.map(externBody) + ], []).filter(Boolean) + ].join('\n'); +} + +function externBody({ exportName, staticNames, instanceNames }) { + return [ + `var ${exportName} = function() {};`, + staticNames.map((staticName) => (isNaN(+staticName) + ? `/** @type {?} */\n${exportName}.${staticName} = function() {};` + : `/** @type {?} */\n${exportName}[${staticName}] = function() {};` + )).join('\n'), + instanceNames.map((instanceName) => (isNaN(+instanceName) + ? `/** @type {?} */\n${exportName}.prototype.${instanceName};` + : `/** @type {?} */\n${exportName}.prototype[${instanceName}];` + )).join('\n') + ].filter(Boolean).join('\n'); +} + +function externsHeader() { + return (`${apacheHeader()} +// @ts-nocheck +/* eslint-disable */ +/** + * @fileoverview Closure Compiler externs for Arrow + * @externs + * @suppress {duplicate,checkTypes} + */ +/** @type {symbol} */ +Symbol.iterator; +/** @type {symbol} */ +Symbol.toPrimitive; +/** @type {symbol} */ +Symbol.asyncIterator; +`); +} + +function getPublicExportedNames(entryModule) { + const fn = function() {}; + const isStaticOrProtoName = (x) => ( + !(x in fn) && + (x !== `default`) && + (x !== `undefined`) && + (x !== `__esModule`) && + (x !== `constructor`) && + !(x.startsWith('_')) + ); + return Object + .getOwnPropertyNames(entryModule) + .filter((name) => name !== 'default') + .filter((name) => ( + typeof entryModule[name] === `object` || + typeof entryModule[name] === `function` + )) + .map((name) => [name, entryModule[name]]) + .reduce((reserved, [name, value]) => { + + const staticNames = value && + typeof value === 'object' ? Object.getOwnPropertyNames(value).filter(isStaticOrProtoName) : + typeof value === 'function' ? Object.getOwnPropertyNames(value).filter(isStaticOrProtoName) : []; + + const instanceNames = (typeof value === `function` && Object.getOwnPropertyNames(value.prototype || {}) || []).filter(isStaticOrProtoName); + + return [...reserved, { exportName: name, staticNames, instanceNames }]; + }, []); +} + +function apacheHeader() { + return `// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License.`; +} diff --git a/src/arrow/js/gulp/compile-task.js b/src/arrow/js/gulp/compile-task.js new file mode 100644 index 000000000..07109ef73 --- /dev/null +++ b/src/arrow/js/gulp/compile-task.js @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { Observable } = require('rxjs'); +const { npmPkgName } = require('./util'); +const { memoizeTask } = require('./memoize-task'); + +const closureTask = require('./closure-task'); +const typescriptTask = require('./typescript-task'); +const { arrowTask, arrowTSTask } = require('./arrow-task'); + +const compileTask = ((cache) => memoizeTask(cache, function compile(target, format, ...args) { + return target === `src` ? Observable.empty() + : target === npmPkgName ? arrowTask(target, format, ...args)() + : target === `ts` ? arrowTSTask(target, format, ...args)() + : format === `umd` ? closureTask(target, format, ...args)() + : typescriptTask(target, format, ...args)(); +}))({}); + +module.exports = compileTask; +module.exports.compileTask = compileTask; diff --git a/src/arrow/js/gulp/memoize-task.js b/src/arrow/js/gulp/memoize-task.js new file mode 100644 index 000000000..408ee3b88 --- /dev/null +++ b/src/arrow/js/gulp/memoize-task.js @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { taskName } = require('./util'); + +const createTask = ((taskFn) => ((target, format, ...args) => { + // Give the memoized fn a displayName so gulp's output is easier to follow. + const fn = () => taskFn(target, format, ...args); + fn.displayName = `${taskFn.name || ``}:${taskName(target, format, ...args)}:task`; + return fn; +})); + +const memoizeTask = ((cache, taskFn) => ((target, format, ...args) => { + // Give the memoized fn a displayName so gulp's output is easier to follow. + const fn = () => ( + cache[taskName(target, format)] || ( + cache[taskName(target, format)] = taskFn(target, format, ...args))); + fn.displayName = `${taskFn.name || ``}:${taskName(target, format, ...args)}:task`; + return fn; +})); + +module.exports = memoizeTask; +module.exports.createTask = createTask; +module.exports.memoizeTask = memoizeTask; diff --git a/src/arrow/js/gulp/package-task.js b/src/arrow/js/gulp/package-task.js new file mode 100644 index 000000000..321e65a30 --- /dev/null +++ b/src/arrow/js/gulp/package-task.js @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { + metadataFiles, packageJSONFields, + mainExport, npmPkgName, npmOrgName, + targetDir, packageName, observableFromStreams +} = require('./util'); + +const gulp = require('gulp'); +const { memoizeTask } = require('./memoize-task'); +const { + ReplaySubject, + EMPTY: ObservableEmpty, + forkJoin: ObservableForkJoin, +} = require('rxjs'); +const { + share +} = require('rxjs/operators'); +const gulpJsonTransform = require('gulp-json-transform'); + +const packageTask = ((cache) => memoizeTask(cache, function bundle(target, format) { + if (target === `src`) return ObservableEmpty(); + const out = targetDir(target, format); + const jsonTransform = gulpJsonTransform(target === npmPkgName ? createMainPackageJson(target, format) : + target === `ts` ? createTypeScriptPackageJson(target, format) + : createScopedPackageJSON(target, format), + 2); + return ObservableForkJoin([ + observableFromStreams(gulp.src(metadataFiles), gulp.dest(out)), // copy metadata files + observableFromStreams(gulp.src(`package.json`), jsonTransform, gulp.dest(out)) // write packageJSONs + ]).pipe(share({ connector: () => new ReplaySubject(), resetOnError: false, resetOnComplete: false, resetOnRefCountZero: false })); +}))({}); + +module.exports = packageTask; +module.exports.packageTask = packageTask; + +// FIXME: set this to false when we have no side effects +const sideEffects = true; + +const createMainPackageJson = (target, format) => (orig) => ({ + ...createTypeScriptPackageJson(target, format)(orig), + bin: orig.bin, + name: npmPkgName, + type: 'commonjs', + main: `${mainExport}.node.js`, + module: `${mainExport}.node.mjs`, + browser: { + [`./${mainExport}.node.js`]: `./${mainExport}.dom.js`, + [`./${mainExport}.node.mjs`]: `./${mainExport}.dom.mjs` + }, + exports: { + import: `./${mainExport}.node.mjs`, + require: `./${mainExport}.node.js`, + }, + types: `${mainExport}.node.d.ts`, + unpkg: `${mainExport}.es2015.min.js`, + jsdelivr: `${mainExport}.es2015.min.js`, + sideEffects: sideEffects, + esm: { mode: `all`, sourceMap: true } +}); + +const createTypeScriptPackageJson = (target, format) => (orig) => ({ + ...createScopedPackageJSON(target, format)(orig), + bin: undefined, + main: `${mainExport}.node.ts`, + module: `${mainExport}.node.ts`, + types: `${mainExport}.node.ts`, + browser: `${mainExport}.dom.ts`, + type: "module", + sideEffects: sideEffects, + esm: { mode: `auto`, sourceMap: true }, + dependencies: { + '@types/flatbuffers': '*', + '@types/node': '*', + ...orig.dependencies + } +}); + +const createScopedPackageJSON = (target, format) => (({ name, ...orig }) => + packageJSONFields.reduce( + (xs, key) => ({ ...xs, [key]: xs[key] || orig[key] }), + { + // un-set version, since it's automatically applied during the release process + version: undefined, + // set the scoped package name (e.g. "@apache-arrow/esnext-esm") + name: `${npmOrgName}/${packageName(target, format)}`, + // set "unpkg"/"jsdeliver" if building scoped UMD target + unpkg: format === 'umd' ? `${mainExport}.js` : undefined, + jsdelivr: format === 'umd' ? `${mainExport}.js` : undefined, + // set "browser" if building scoped UMD target, otherwise "Arrow.dom" + browser: format === 'umd' ? `${mainExport}.js` : `${mainExport}.dom.js`, + // set "main" to "Arrow" if building scoped UMD target, otherwise "Arrow.node" + main: format === 'umd' ? `${mainExport}.js` : `${mainExport}.node.js`, + // set "type" to `module` or `commonjs` (https://nodejs.org/api/packages.html#packages_type) + type: format === 'esm' ? `module` : `commonjs`, + // set "module" if building scoped ESM target + module: format === 'esm' ? `${mainExport}.node.js` : undefined, + // set "sideEffects" to false as a hint to Webpack that it's safe to tree-shake the ESM target + sideEffects: format === 'esm' ? sideEffects : undefined, + // include "esm" settings for https://www.npmjs.com/package/esm if building scoped ESM target + esm: format === `esm` ? { mode: `auto`, sourceMap: true } : undefined, + // set "types" (for TypeScript/VSCode) + types: format === 'umd' ? undefined : `${mainExport}.node.d.ts`, + } + ) +); diff --git a/src/arrow/js/gulp/test-task.js b/src/arrow/js/gulp/test-task.js new file mode 100644 index 000000000..2012f7429 --- /dev/null +++ b/src/arrow/js/gulp/test-task.js @@ -0,0 +1,186 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const del = require('del'); +const path = require('path'); +const mkdirp = require('mkdirp'); +const cpy = require('cpy'); +const { argv } = require('./argv'); +const { promisify } = require('util'); +const glob = promisify(require('glob')); +const child_process = require(`child_process`); +const { memoizeTask } = require('./memoize-task'); +const readFile = promisify(require('fs').readFile); +const asyncDone = promisify(require('async-done')); +const exec = promisify(require('child_process').exec); +const parseXML = promisify(require('xml2js').parseString); +const { targetAndModuleCombinations, npmPkgName } = require('./util'); + +const jestArgv = [`--reporters=jest-silent-reporter`]; + +if (argv.verbose) { + jestArgv.push(`--verbose`); +} + +if (targetAndModuleCombinations.length > 1) { + jestArgv.push(`--detectOpenHandles`); +} + +const jest = path.join(path.parse(require.resolve(`jest`)).dir, `../bin/jest.js`); +const testOptions = { + stdio: [`ignore`, `inherit`, `inherit`], + env: { + ...process.env, + // hide fs.promises/stream[Symbol.asyncIterator] warnings + NODE_NO_WARNINGS: `1`, + }, +}; + +const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function test(target, format) { + const opts = { ...testOptions }; + const args = [...execArgv]; + if (format === 'esm' || target === 'ts' || target === 'src' || target === npmPkgName) { + args.unshift(`--experimental-vm-modules`); + } + if (argv.coverage) { + args.push(`-c`, `jestconfigs/jest.coverage.config.js`); + } else { + const cfgname = [target, format].filter(Boolean).join('.'); + args.push(`-c`, `jestconfigs/jest.${cfgname}.config.js`, `test/unit/`); + } + opts.env = { + ...opts.env, + TEST_TARGET: target, + TEST_MODULE: format, + TEST_DOM_STREAMS: (target ==='src' || format === 'umd').toString(), + TEST_NODE_STREAMS: (target ==='src' || format !== 'umd').toString(), + TEST_TS_SOURCE: !!argv.coverage || (target === 'src') || (opts.env.TEST_TS_SOURCE === 'true') + }; + return asyncDone(() => child_process.spawn(`node`, args, opts)); +}))({}, [jest, ...jestArgv], testOptions); + +module.exports = testTask; +module.exports.testTask = testTask; +module.exports.cleanTestData = cleanTestData; +module.exports.createTestData = createTestData; + +// Pull C++ and Java paths from environment vars first, otherwise sane defaults +const ARROW_HOME = process.env.ARROW_HOME || path.resolve('../'); +const ARROW_JAVA_DIR = process.env.ARROW_JAVA_DIR || path.join(ARROW_HOME, 'java'); +const CPP_EXE_PATH = process.env.ARROW_CPP_EXE_PATH || path.join(ARROW_HOME, 'cpp/build/debug'); +const ARROW_INTEGRATION_DIR = process.env.ARROW_INTEGRATION_DIR || path.join(ARROW_HOME, 'integration'); +const CPP_JSON_TO_ARROW = path.join(CPP_EXE_PATH, 'arrow-json-integration-test'); +const CPP_FILE_TO_STREAM = path.join(CPP_EXE_PATH, 'arrow-file-to-stream'); + +const testFilesDir = path.join(ARROW_HOME, 'js/test/data'); +const snapshotsDir = path.join(ARROW_HOME, 'js/test/__snapshots__'); +const cppFilesDir = path.join(testFilesDir, 'cpp'); +const javaFilesDir = path.join(testFilesDir, 'java'); +const jsonFilesDir = path.join(testFilesDir, 'json'); + +async function cleanTestData() { + return await del([ + `${cppFilesDir}/**`, + `${javaFilesDir}/**`, + `${jsonFilesDir}/**`, + `${snapshotsDir}/**` + ]); +} + +async function createTestJSON() { + await mkdirp(jsonFilesDir); + await cpy(`cp ${ARROW_INTEGRATION_DIR}/data/*.json`, jsonFilesDir); + await exec(`python3 ${ARROW_INTEGRATION_DIR}/integration_test.py --write_generated_json ${jsonFilesDir}`); +} + +async function createTestData() { + + let JAVA_TOOLS_JAR = process.env.ARROW_JAVA_INTEGRATION_JAR; + if (!JAVA_TOOLS_JAR) { + const pom_version = await + readFile(path.join(ARROW_JAVA_DIR, 'pom.xml')) + .then((pom) => parseXML(pom.toString())) + .then((pomXML) => pomXML.project.version[0]); + JAVA_TOOLS_JAR = path.join(ARROW_JAVA_DIR, `/tools/target/arrow-tools-${pom_version}-jar-with-dependencies.jar`); + } + + await cleanTestData().then(createTestJSON); + await mkdirp(path.join(cppFilesDir, 'file')); + await mkdirp(path.join(javaFilesDir, 'file')); + await mkdirp(path.join(cppFilesDir, 'stream')); + await mkdirp(path.join(javaFilesDir, 'stream')); + + const errors = []; + const names = await glob(path.join(jsonFilesDir, '*.json')); + + for (let jsonPath of names) { + const name = path.parse(path.basename(jsonPath)).name; + const arrowCppFilePath = path.join(cppFilesDir, 'file', `${name}.arrow`); + const arrowJavaFilePath = path.join(javaFilesDir, 'file', `${name}.arrow`); + const arrowCppStreamPath = path.join(cppFilesDir, 'stream', `${name}.arrow`); + const arrowJavaStreamPath = path.join(javaFilesDir, 'stream', `${name}.arrow`); + try { + await generateCPPFile(path.resolve(jsonPath), arrowCppFilePath); + await generateCPPStream(arrowCppFilePath, arrowCppStreamPath); + } catch (e) { errors.push(`${e.stdout}\n${e.message}`); } + try { + await generateJavaFile(path.resolve(jsonPath), arrowJavaFilePath); + await generateJavaStream(arrowJavaFilePath, arrowJavaStreamPath); + } catch (e) { errors.push(`${e.stdout}\n${e.message}`); } + } + if (errors.length) { + console.error(errors.join(`\n`)); + process.exit(1); + } + + async function generateCPPFile(jsonPath, filePath) { + await del(filePath); + return await exec( + `${CPP_JSON_TO_ARROW} ${ + `--integration --mode=JSON_TO_ARROW`} ${ + `--json=${jsonPath} --arrow=${filePath}`}`, + { maxBuffer: Math.pow(2, 53) - 1 } + ); + } + + async function generateCPPStream(filePath, streamPath) { + await del(streamPath); + return await exec( + `${CPP_FILE_TO_STREAM} ${filePath} > ${streamPath}`, + { maxBuffer: Math.pow(2, 53) - 1 } + ); + } + + async function generateJavaFile(jsonPath, filePath) { + await del(filePath); + return await exec( + `java -cp ${JAVA_TOOLS_JAR} ${ + `org.apache.arrow.tools.Integration -c JSON_TO_ARROW`} ${ + `-j ${path.resolve(jsonPath)} -a ${filePath}`}`, + { maxBuffer: Math.pow(2, 53) - 1 } + ); + } + + async function generateJavaStream(filePath, streamPath) { + await del(streamPath); + return await exec( + `java -cp ${JAVA_TOOLS_JAR} ${ + `org.apache.arrow.tools.FileToStream`} ${filePath} ${streamPath}`, + { maxBuffer: Math.pow(2, 53) - 1 } + ); + } +} diff --git a/src/arrow/js/gulp/typescript-task.js b/src/arrow/js/gulp/typescript-task.js new file mode 100644 index 000000000..ed03b8453 --- /dev/null +++ b/src/arrow/js/gulp/typescript-task.js @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const { + targetDir, + tsconfigName, + observableFromStreams, + shouldRunInChildProcess, + spawnGulpCommandInChildProcess, +} = require('./util'); + +const gulp = require('gulp'); +const path = require('path'); +const ts = require(`gulp-typescript`); +const sourcemaps = require('gulp-sourcemaps'); +const { memoizeTask } = require('./memoize-task'); +const { + ReplaySubject, + forkJoin: ObservableForkJoin, +} = require('rxjs'); +const { + mergeWith, + takeLast, + share +} = require('rxjs/operators'); + +const typescriptTask = ((cache) => memoizeTask(cache, function typescript(target, format) { + if (shouldRunInChildProcess(target, format)) { + return spawnGulpCommandInChildProcess('compile', target, format); + } + + const out = targetDir(target, format); + const tsconfigPath = path.join(`tsconfig`, `tsconfig.${tsconfigName(target, format)}.json`); + return compileTypescript(out, tsconfigPath) + .pipe(mergeWith(compileBinFiles(target, format))) + .pipe(takeLast(1)) + .pipe(share({ connector: () => new ReplaySubject(), resetOnError: false, resetOnComplete: false, resetOnRefCountZero: false })) +}))({}); + +function compileBinFiles(target, format) { + const out = targetDir(target, format); + const tsconfigPath = path.join(`tsconfig`, `tsconfig.${tsconfigName('bin', 'cjs')}.json`); + return compileTypescript(path.join(out, 'bin'), tsconfigPath, { target }); +} + +function compileTypescript(out, tsconfigPath, tsconfigOverrides) { + const tsProject = ts.createProject(tsconfigPath, { typescript: require(`typescript`), ...tsconfigOverrides}); + const { stream: { js, dts } } = observableFromStreams( + tsProject.src(), sourcemaps.init(), + tsProject(ts.reporter.defaultReporter()) + ); + const writeSources = observableFromStreams(tsProject.src(), gulp.dest(path.join(out, 'src'))); + const writeDTypes = observableFromStreams(dts, sourcemaps.write('./', { includeContent: false, sourceRoot: 'src' }), gulp.dest(out)); + const mapFile = tsProject.options.module === 5 ? esmMapFile : cjsMapFile; + const writeJS = observableFromStreams(js, sourcemaps.write('./', { mapFile, includeContent: false }), gulp.dest(out)); + return ObservableForkJoin([writeSources, writeDTypes, writeJS]); +} + +function cjsMapFile(mapFilePath) { return mapFilePath; } +function esmMapFile(mapFilePath) { return mapFilePath.replace('.js.map', '.mjs.map'); } + +module.exports = typescriptTask; +module.exports.typescriptTask = typescriptTask; +module.exports.compileBinFiles = compileBinFiles; diff --git a/src/arrow/js/gulp/util.js b/src/arrow/js/gulp/util.js new file mode 100644 index 000000000..d8cde29e8 --- /dev/null +++ b/src/arrow/js/gulp/util.js @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const fs = require('fs'); +const path = require(`path`); +const pump = require(`stream`).pipeline; +const child_process = require(`child_process`); +const { targets, modules } = require('./argv'); +const { + ReplaySubject, + empty: ObservableEmpty, + throwError: ObservableThrow, + fromEvent: ObservableFromEvent +} = require('rxjs'); +const { + share, + flatMap, + takeUntil, + defaultIfEmpty, + mergeWith, +} = require('rxjs/operators'); +const asyncDone = require('util').promisify(require('async-done')); + +const mainExport = `Arrow`; +const npmPkgName = `apache-arrow`; +const npmOrgName = `@${npmPkgName}`; + +const releasesRootDir = `targets`; +const knownTargets = [`es5`, `es2015`, `esnext`]; +const knownModules = [`cjs`, `esm`, `cls`, `umd`]; +const tasksToSkipPerTargetOrFormat = { + src: { clean: true, build: true }, + cls: { test: true, package: true } +}; +const packageJSONFields = [ + `version`, `license`, `description`, + `author`, `homepage`, `repository`, + `bugs`, `keywords`, `dependencies`, + `bin` +]; + +const metadataFiles = [`LICENSE.txt`, `NOTICE.txt`, `README.md`].map((filename) => { + let prefixes = [`./`, `../`]; + let p = prefixes.find((prefix) => { + try { + fs.statSync(path.resolve(path.join(prefix, filename))); + } catch (e) { return false; } + return true; + }); + if (!p) { + throw new Error(`Couldn't find ${filename} in ./ or ../`); + } + return path.join(p, filename); +}); + +// see: https://github.com/google/closure-compiler/blob/c1372b799d94582eaf4b507a4a22558ff26c403c/src/com/google/javascript/jscomp/CompilerOptions.java#L2988 +const gCCLanguageNames = { + es5: `ECMASCRIPT5`, + es2015: `ECMASCRIPT_2015`, + es2016: `ECMASCRIPT_2016`, + es2017: `ECMASCRIPT_2017`, + es2018: `ECMASCRIPT_2018`, + es2019: `ECMASCRIPT_2019`, + esnext: `ECMASCRIPT_NEXT` +}; + +function taskName(target, format) { + return !format ? target : `${target}:${format}`; +} + +function packageName(target, format) { + return !format ? target : `${target}-${format}`; +} + +function tsconfigName(target, format) { + return !format ? target : `${target}.${format}`; +} + +function targetDir(target, format) { + return path.join(releasesRootDir, ...(!format ? [target] : [target, format])); +} + +function shouldRunInChildProcess(target, format) { + // If we're building more than one module/target, then yes run this task in a child process + if (targets.length > 1 || modules.length > 1) { return true; } + // If the target we're building *isn't* the target the gulp command was configured to run, then yes run that in a child process + if (targets[0] !== target || modules[0] !== format) { return true; } + // Otherwise no need -- either gulp was run for just one target, or we've been spawned as the child of a multi-target parent gulp + return false; +} + +const gulp = path.join(path.parse(require.resolve(`gulp`)).dir, `bin/gulp.js`); +function spawnGulpCommandInChildProcess(command, target, format) { + const args = [gulp, command, '-t', target, '-m', format, `--silent`]; + const opts = { + stdio: [`ignore`, `inherit`, `inherit`], + env: { ...process.env, NODE_NO_WARNINGS: `1` } + }; + return asyncDone(() => child_process.spawn(`node`, args, opts)) + .catch((e) => { throw `Error in "${command}:${taskName(target, format)}" task`; }); +} + +const logAndDie = (e) => { if (e) { process.exit(1) } }; +function observableFromStreams(...streams) { + if (streams.length <= 0) { return ObservableEmpty(); } + const pumped = streams.length <= 1 ? streams[0] : pump(...streams, logAndDie); + const fromEvent = ObservableFromEvent.bind(null, pumped); + const streamObs = fromEvent(`data`).pipe( + mergeWith(fromEvent(`error`).pipe(flatMap((e) => ObservableThrow(e)))), + takeUntil(fromEvent(`end`).pipe(mergeWith(fromEvent(`close`)))), + defaultIfEmpty(`empty stream`), + share({ connector: () => new ReplaySubject(), resetOnError: false, resetOnComplete: false, resetOnRefCountZero: false }) + ); + streamObs.stream = pumped; + streamObs.observable = streamObs; + return streamObs; +} + +function* combinations(_targets, _modules) { + const targets = known(knownTargets, _targets || [`all`]); + const modules = known(knownModules, _modules || [`all`]); + + if (_targets.includes(`src`)) { + yield [`src`, ``]; + return; + } + + if (_targets.includes(`all`) && _modules.includes(`all`)) { + yield [`ts`, ``]; + yield [`src`, ``]; + yield [npmPkgName, ``]; + } + + for (const format of modules) { + for (const target of targets) { + yield [target, format]; + } + } + + function known(known, values) { + return values.includes(`all`) ? known + : values.includes(`src`) ? [`src`] + : Object.keys( + values.reduce((map, arg) => (( + (known.includes(arg)) && + (map[arg.toLowerCase()] = true) + || true) && map + ), {}) + ).sort((a, b) => known.indexOf(a) - known.indexOf(b)); + } +} + +const publicModulePaths = (dir) => [ + `${dir}/${mainExport}.dom.js`, + `${dir}/util/int.js`, + `${dir}/compute/predicate.js`, +]; + +const esmRequire = require(`esm`)(module, { + mode: `auto`, + cjs: { + /* A boolean for storing ES modules in require.cache. */ + cache: true, + /* A boolean for respecting require.extensions in ESM. */ + extensions: true, + /* A boolean for __esModule interoperability. */ + interop: true, + /* A boolean for importing named exports of CJS modules. */ + namedExports: true, + /* A boolean for following CJS path rules in ESM. */ + paths: true, + /* A boolean for __dirname, __filename, and require in ESM. */ + vars: true, + } +}); + +module.exports = { + mainExport, npmPkgName, npmOrgName, metadataFiles, packageJSONFields, + + knownTargets, knownModules, tasksToSkipPerTargetOrFormat, gCCLanguageNames, + + taskName, packageName, tsconfigName, targetDir, combinations, observableFromStreams, + publicModulePaths, esmRequire, shouldRunInChildProcess, spawnGulpCommandInChildProcess, + + targetAndModuleCombinations: [...combinations(targets, modules)] +}; |