diff options
Diffstat (limited to 'fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload')
30 files changed, 2424 insertions, 0 deletions
diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/CMakeLists.txt b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/CMakeLists.txt new file mode 100644 index 00000000..667f0b4e --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/CMakeLists.txt @@ -0,0 +1,116 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project(wasm_workloads) + +####################################### +add_subdirectory(bwa) +add_subdirectory(meshoptimizer) +add_subdirectory(wasm-av1) + +####################################### +include(ExternalProject) + +################ iwasm ################ +ExternalProject_Add(iwasm + PREFIX + iwasm-build + BUILD_ALWAYS + YES + SOURCE_DIR + ${CMAKE_CURRENT_SOURCE_DIR}/../../product-mini/platforms/linux + CONFIGURE_COMMAND + ${CMAKE_COMMAND} -S ${CMAKE_CURRENT_SOURCE_DIR}/../../product-mini/platforms/linux -B build -DWAMR_BUILD_LIBC_EMCC=1 + BUILD_COMMAND + ${CMAKE_COMMAND} --build build --parallel 4 + INSTALL_COMMAND + # FIXME: replace with --install + ${CMAKE_COMMAND} -E copy_if_different + ${CMAKE_CURRENT_BINARY_DIR}/iwasm-build/src/iwasm-build/build/iwasm + ${CMAKE_CURRENT_BINARY_DIR}/iwasm +) + +################ wamrc ################ +ExternalProject_Add(wamrc + PREFIX + wamrc-build + BUILD_ALWAYS + YES + SOURCE_DIR + ${CMAKE_CURRENT_SOURCE_DIR}/../../wamr-compiler + CONFIGURE_COMMAND + ${CMAKE_COMMAND} -S ${CMAKE_CURRENT_SOURCE_DIR}/../../wamr-compiler -B build + BUILD_COMMAND + ${CMAKE_COMMAND} --build build --parallel 4 + INSTALL_COMMAND + # FIXME: replace with --install + ${CMAKE_COMMAND} -E copy_if_different + ${CMAKE_CURRENT_BINARY_DIR}/wamrc-build/src/wamrc-build/build/wamrc + ${CMAKE_CURRENT_BINARY_DIR}/wamrc +) + +################ .aot ################ +add_custom_target( + bwa_to_aot + ALL + DEPENDS + bwa wamrc + COMMAND + ./wamrc -o bwa.aot ./bwa/bwa.wasm + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_custom_target( + codecbench_to_aot + ALL + DEPENDS + codecbench wamrc + COMMAND + ./wamrc -o codecbench.aot ./meshoptimizer/codecbench.wasm + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_custom_target( + av1_to_aot + ALL + DEPENDS + av1 wamrc + COMMAND + ./wamrc -o testavx.aot ./wasm-av1/testavx.opt.wasm + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +################ smoking test ################ +include(CTest) + +add_test( + NAME + run_bwa + COMMAND + ./iwasm --dir=. ./bwa.aot index ./bwa/hs38DH-extra.fa + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_test( + NAME + run_codecbench + COMMAND + ./iwasm codecbench.aot + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_test( + NAME + run_av1 + COMMAND + ./iwasm --dir=. testavx.aot ./wasm-av1/elephants_dream_480p24.ivf + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/README.md b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/README.md new file mode 100644 index 00000000..0e6a3a41 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/README.md @@ -0,0 +1,34 @@ +All workloads have similar requirment of software dependencies, including **emsdk** and **binaryen** + +> There might be slight differences when using MacOS and other Linux distro than Ubuntu. This document targets +Ubuntu 20.04 as an example. + +## Installation instructions + +use [preparation.sh](./preparation.sh) to install all dependencies before compiling any workload. Or use [*vscode DevContainer*](../../.devcontainer/) + +The script installs below software: + +- **emsdk**. Refer to [the guide](https://emscripten.org/docs/getting_started/downloads.html). Don't forget to activate + emsdk and set up environment variables. Verify it with `echo ${EMSDK}`. Please be sure to install and activate the building + of 3.0.0 + +``` bash +$ cd /opt +$ git clone https://github.com/emscripten-core/emsdk.git +$ cd emsdk +$ git pull +$ ./emsdk install 3.0.0 +$ ./emsdk activate 3.0.0 +$ echo "source /opt/emsdk/emsdk_env.sh" >> "${HOME}"/.bashrc +``` + +- **binaryen**. Install + [latest release](https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-linux.tar.gz) + to */opt/binaryen* + +``` bash +$ wget https://github.com/WebAssembly/binaryen/releases/download/${BINARYEN_VER}/${BINARYEN_FILE} +$ tar zxf ${BINARYEN_FILE} -C /opt +$ ln -sf /opt/binaryen-${BINARYEN_VER} /opt/binaryen +``` diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/XNNPACK/.gitignore b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/XNNPACK/.gitignore new file mode 100644 index 00000000..09f3fe92 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/XNNPACK/.gitignore @@ -0,0 +1,2 @@ +xnnpack +build diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/XNNPACK/CMakeLists.txt b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/XNNPACK/CMakeLists.txt new file mode 100644 index 00000000..aef138d5 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/XNNPACK/CMakeLists.txt @@ -0,0 +1,147 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.0) + +project(xnnpack_wasm) + +################ EMCC ################ +include(ExternalProject) + +ExternalProject_Add(xnnpack + PREFIX xnnpack + GIT_REPOSITORY https://github.com/google/XNNPACK.git + GIT_TAG 4570a7151aa4f3e57eca14a575eeff6bb13e26be + GIT_PROGRESS ON + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack + UPDATE_COMMAND git restore . + && cmake -E copy ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack/google3/third_party/XNNPACK/microkernels.bzl + ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack/ + && git apply ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack.patch + CONFIGURE_COMMAND "" + # grep xnnpack_benchmark -A 1 BUILD.bazel \ + # | grep "name =" \ + # | awk '{print $3}' \ + # | sed -e 's/\"//g' -e 's/,//g' -e 's/^/\/\/:/g' + BUILD_COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack + && bazel --output_user_root=build-user-output build -c opt --config=wasm + //:qs8_dwconv_bench.wasm + //:qs8_f32_vcvt_bench.wasm + //:qs8_gemm_bench.wasm + //:qs8_requantization_bench.wasm + //:qs8_vadd_bench.wasm + //:qs8_vaddc_bench.wasm + //:qs8_vcvt_bench.wasm + //:qs8_vlrelu_bench.wasm + //:qs8_vmul_bench.wasm + //:qs8_vmulc_bench.wasm + //:qu8_f32_vcvt_bench.wasm + //:qu8_gemm_bench.wasm + //:qu8_requantization_bench.wasm + //:qu8_vadd_bench.wasm + //:qu8_vaddc_bench.wasm + //:qu8_vcvt_bench.wasm + //:qu8_vlrelu_bench.wasm + //:qu8_vmul_bench.wasm + //:qu8_vmulc_bench.wasm + //:bf16_gemm_bench.wasm + //:f16_igemm_bench.wasm + //:f16_gemm_bench.wasm + //:f16_raddstoreexpminusmax_bench.wasm + //:f16_spmm_bench.wasm + //:f16_vsigmoid_bench.wasm + //:f16_f32_vcvt_bench.wasm + //:f32_igemm_bench.wasm + //:f32_conv_hwc_bench.wasm + //:f16_conv_hwc2chw_bench.wasm + //:f16_gavgpool_cw_bench.wasm + //:f32_gavgpool_cw_bench.wasm + //:f32_conv_hwc2chw_bench.wasm + //:f16_dwconv_bench.wasm + //:f32_dwconv_bench.wasm + //:f32_dwconv2d_chw_bench.wasm + //:f16_dwconv2d_chw_bench.wasm + //:f32_f16_vcvt_bench.wasm + //:xx_transpose_bench.wasm + //:x8_transpose_bench.wasm + //:x16_transpose_bench.wasm + //:x24_transpose_bench.wasm + //:x32_transpose_bench.wasm + //:x64_transpose_bench.wasm + //:f32_gemm_bench.wasm + //:f32_qs8_vcvt_bench.wasm + //:f32_qu8_vcvt_bench.wasm + //:f32_raddexpminusmax_bench.wasm + //:f32_raddextexp_bench.wasm + //:f32_raddstoreexpminusmax_bench.wasm + //:f32_rmax_bench.wasm + //:f32_spmm_bench.wasm + //:f32_softmax_bench.wasm + //:f16_velu_bench.wasm + //:f32_velu_bench.wasm + //:f32_vhswish_bench.wasm + //:f32_vlrelu_bench.wasm + //:f32_vrelu_bench.wasm + //:f32_vscaleexpminusmax_bench.wasm + //:f32_vscaleextexp_bench.wasm + //:f32_vsigmoid_bench.wasm + //:f16_vsqrt_bench.wasm + //:f32_vsqrt_bench.wasm + //:f32_im2col_gemm_bench.wasm + //:rounding_bench.wasm + //:s16_rmaxabs_bench.wasm + //:s16_window_bench.wasm + //:u32_filterbank_accumulate_bench.wasm + //:u32_filterbank_subtract_bench.wasm + //:u32_vlog_bench.wasm + //:u64_u32_vsqrtshift_bench.wasm + //:i16_vlshift_bench.wasm + //:cs16_vsquareabs_bench.wasm + //:cs16_bfly4_bench.wasm + //:cs16_fftr_bench.wasm + //:x8_lut_bench.wasm + //:abs_bench.wasm + //:average_pooling_bench.wasm + //:bankers_rounding_bench.wasm + //:ceiling_bench.wasm + //:channel_shuffle_bench.wasm + //:convert_bench.wasm + //:convolution_bench.wasm + //:deconvolution_bench.wasm + //:elu_bench.wasm + //:floor_bench.wasm + //:global_average_pooling_bench.wasm + //:hardswish_bench.wasm + //:leaky_relu_bench.wasm + //:max_pooling_bench.wasm + //:negate_bench.wasm + //:sigmoid_bench.wasm + //:prelu_bench.wasm + //:softmax_bench.wasm + //:square_bench.wasm + //:square_root_bench.wasm + //:truncation_bench.wasm + //:f16_gemm_e2e_bench.wasm + //:f32_dwconv_e2e_bench.wasm + //:f32_gemm_e2e_bench.wasm + //:qs8_dwconv_e2e_bench.wasm + //:qs8_gemm_e2e_bench.wasm + //:qu8_gemm_e2e_bench.wasm + //:qu8_dwconv_e2e_bench.wasm + //:end2end_bench.wasm + //:f16_exp_ulp_eval.wasm + //:f16_expminus_ulp_eval.wasm + //:f16_expm1minus_ulp_eval.wasm + //:f16_sigmoid_ulp_eval.wasm + //:f16_sqrt_ulp_eval.wasm + //:f32_exp_ulp_eval.wasm + //:f32_expminus_ulp_eval.wasm + //:f32_expm1minus_ulp_eval.wasm + //:f32_extexp_ulp_eval.wasm + //:f32_sigmoid_ulp_eval.wasm + //:f32_sqrt_ulp_eval.wasm + //:f32_tanh_ulp_eval.wasm + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory + ${CMAKE_CURRENT_SOURCE_DIR}/xnnpack/bazel-out/wasm-opt/bin/ + ${CMAKE_BINARY_DIR}/wasm-opt +) diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/XNNPACK/README.md b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/XNNPACK/README.md new file mode 100644 index 00000000..7984d9ce --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/XNNPACK/README.md @@ -0,0 +1,48 @@ +"XNNPACK" sample introduction +============== + +This sample demonstrates how to build [XNNPACK](https://github.com/google/XNNPACK) benchmarks into WebAssembly with emsdk toolchain and run them with iwasm. + +## Installation toolchains + +please refer to [installation instructions](../README.md). + +## Build XNNPACK + +```bash +cd <wamr-dir>/samples/workload/XNNPACK +mkdir build +cd build +cmake .. +``` +The wasm files are generated under folder samples/workload/XNNPACK/xnnpack/bazel-bin. + +## Run benchmarks + +Firstly please build iwasm with simd, libc-emcc and lib-pthread support: + +``` bash +$ cd <wamr-dir>/product-mini/platforms/linux/ +$ mkdir build && cd build +$ cmake .. -DWAMR_BUILD_LIBC_EMCC=1 -DWAMR_BUILD_LIB_PTHREAD=1 +$ make +``` + +And please build wamrc: + +``` bash +cd <wamr-dir>/wamr-compiler +./build_llvm.sh +mkdir build && cd build +cmake .. +make +``` + +Then compile wasm file to aot file and run: + +``` shell +$ cd <wamr-dir>/samples/workload/XNNPACK/xnnpack/bazel-bin +$ wamrc -o average_pooling_bench.aot average_pooling_bench.wasm (or other wasm files) +$ iwasm average_pooling_bench.aot +``` + diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/XNNPACK/benchmark.patch b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/XNNPACK/benchmark.patch new file mode 100644 index 00000000..713b476d --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/XNNPACK/benchmark.patch @@ -0,0 +1,14 @@ +diff --git include/benchmark/benchmark.h include/benchmark/benchmark.h +index 9b54802..baa5938 100755 +--- include/benchmark/benchmark.h ++++ include/benchmark/benchmark.h +@@ -364,7 +364,9 @@ template <class Tp> + inline BENCHMARK_ALWAYS_INLINE void DoNotOptimize(Tp const& value) { + internal::UseCharPointer(&reinterpret_cast<char const volatile&>(value)); + } ++ + // FIXME Add ClobberMemory() for non-gnu and non-msvc compilers ++inline BENCHMARK_ALWAYS_INLINE void ClobberMemory() { } + #endif + + // This class is used for user-defined counters. diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/XNNPACK/xnnpack.patch b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/XNNPACK/xnnpack.patch new file mode 100644 index 00000000..3fb6b230 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/XNNPACK/xnnpack.patch @@ -0,0 +1,141 @@ +diff --git a/.bazelrc b/.bazelrc +index 688279da1..376996885 100644 +--- a/.bazelrc ++++ b/.bazelrc +@@ -53,4 +53,9 @@ build:ios_fat --watchos_cpus=armv7k + build:macos --apple_platform_type=macos + + build:macos_arm64 --config=macos +-build:macos_arm64 --cpu=darwin_arm64 +\ No newline at end of file ++build:macos_arm64 --cpu=darwin_arm64 ++ ++build:wasm --cpu=wasm ++build:wasm --features=wasm_simd ++build:wasm --crosstool_top=@emsdk//emscripten_toolchain:everything ++build:wasm --host_crosstool_top=@bazel_tools//tools/cpp:toolchain +diff --git a/WORKSPACE b/WORKSPACE +index cd8960ffa..787e03ca8 100644 +--- a/WORKSPACE ++++ b/WORKSPACE +@@ -29,8 +29,9 @@ http_archive( + # Google Benchmark library, used in micro-benchmarks. + http_archive( + name = "com_google_benchmark", +- strip_prefix = "benchmark-main", +- urls = ["https://github.com/google/benchmark/archive/main.zip"], ++ sha256 = "1ba14374fddcd9623f126b1a60945e4deac4cdc4fb25a5f25e7f779e36f2db52", ++ strip_prefix = "benchmark-d2a8a4ee41b923876c034afb939c4fc03598e622", ++ urls = ["https://github.com/google/benchmark/archive/d2a8a4ee41b923876c034afb939c4fc03598e622.zip"], + ) + + # FP16 library, used for half-precision conversions +@@ -92,8 +93,25 @@ http_archive( + ], + ) + ++load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") ++http_archive( ++ name = "emsdk", ++ # Use emsdk-3.0.0 since the larger version may: ++ # - compress the wasm file into a tar file but not directly generate wasm file ++ # - generate incomplete implementation of libc API, e.g. throw exception in getentropy ++ strip_prefix = "emsdk-3.0.0/bazel", ++ url = "https://github.com/emscripten-core/emsdk/archive/refs/tags/3.0.0.tar.gz", ++ sha256 = "a41dccfd15be9e85f923efaa0ac21943cbab77ec8d39e52f25eca1ec61a9ac9e" ++) ++ ++load("@emsdk//:deps.bzl", emsdk_deps = "deps") ++emsdk_deps() ++ ++load("@emsdk//:emscripten_deps.bzl", emsdk_emscripten_deps = "emscripten_deps") ++emsdk_emscripten_deps() ++ + # Android NDK location and version is auto-detected from $ANDROID_NDK_HOME environment variable +-android_ndk_repository(name = "androidndk") ++#android_ndk_repository(name = "androidndk") + + # Android SDK location and API is auto-detected from $ANDROID_HOME environment variable +-android_sdk_repository(name = "androidsdk") ++#android_sdk_repository(name = "androidsdk") +diff --git a/build_defs.bzl b/build_defs.bzl +index b8217a18d..6f2d1675e 100644 +--- a/build_defs.bzl ++++ b/build_defs.bzl +@@ -380,7 +380,7 @@ def xnnpack_benchmark(name, srcs, copts = [], deps = [], tags = []): + explicitly specified. + """ + native.cc_binary( +- name = name, ++ name = name + ".wasm", + srcs = srcs, + copts = xnnpack_std_cxxopts() + [ + "-Iinclude", +@@ -405,5 +405,5 @@ def xnnpack_benchmark(name, srcs, copts = [], deps = [], tags = []): + ":emscripten": xnnpack_emscripten_deps(), + "//conditions:default": [], + }), +- tags = tags, ++ tags = tags, + ) +diff --git a/emscripten.bzl b/emscripten.bzl +index f1557a7b1..7f964a094 100644 +--- a/emscripten.bzl ++++ b/emscripten.bzl +@@ -25,12 +25,19 @@ def xnnpack_emscripten_benchmark_linkopts(): + """Emscripten-specific linkopts for benchmarks.""" + return [ + "-s ASSERTIONS=1", +- "-s ENVIRONMENT=node,shell,web", +- "-s ERROR_ON_UNDEFINED_SYMBOLS=1", +- "-s EXIT_RUNTIME=1", ++ "-s ERROR_ON_UNDEFINED_SYMBOLS=0", + "-s ALLOW_MEMORY_GROWTH=1", + "-s TOTAL_MEMORY=536870912", # 512M +- "--pre-js $(location :preamble.js.lds)", ++ "-s USE_PTHREADS=0", ++ "-s STANDALONE_WASM=1", ++ "-Wno-unused", ++ "-Wno-unused-variable", ++ "-Wno-unused-command-line-argument", ++ "-Wl,--export=__heap_base", ++ "-Wl,--export=__data_end", ++ "-Wl,--export=malloc", ++ "-Wl,--export=free", ++ "--oformat=wasm", + ] + + def xnnpack_emscripten_deps(): +diff --git a/src/log.c b/src/log.c +index 5715f2f85..4b3e4261b 100644 +--- a/src/log.c ++++ b/src/log.c +@@ -55,7 +55,7 @@ + #endif + + #if XNN_LOG_TO_STDIO +-static void xnn_vlog(int output_handle, const char* prefix, size_t prefix_length, const char* format, va_list args) { ++void xnn_vlog(int output_handle, const char* prefix, size_t prefix_length, const char* format, va_list args) { + char stack_buffer[XNN_LOG_STACK_BUFFER_SIZE]; + char* heap_buffer = NULL; + char* out_buffer = &stack_buffer[0]; +diff --git a/third_party/cpuinfo.BUILD b/third_party/cpuinfo.BUILD +index 1997f4e3a..5e03c43af 100644 +--- a/third_party/cpuinfo.BUILD ++++ b/third_party/cpuinfo.BUILD +@@ -150,7 +150,7 @@ cc_library( + "src/arm/midr.h", + ], + deps = [ +- "@clog", ++ "//deps/clog" + ], + ) + +@@ -352,5 +352,5 @@ config_setting( + + config_setting( + name = "emscripten", +- values = {"crosstool_top": "//toolchain:emscripten"}, ++ values = {"crosstool_top": "@emsdk//emscripten_toolchain:everything"}, + ) diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/bwa/.gitignore b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/bwa/.gitignore new file mode 100644 index 00000000..cd720959 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/bwa/.gitignore @@ -0,0 +1,4 @@ +build +libz +bwa +include diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/bwa/CMakeLists.bwa_wasm.txt b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/bwa/CMakeLists.bwa_wasm.txt new file mode 100644 index 00000000..a8d1d882 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/bwa/CMakeLists.bwa_wasm.txt @@ -0,0 +1,124 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project(bwa_wasm C) + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../cmake) + +################ dependencies ################ +find_package(Binaryen 111 REQUIRED) + +################ LIBZ ################ +set(LIBZ_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../libz) +add_library(z_wasm STATIC + ${LIBZ_SRC_DIR}/adler32.c + ${LIBZ_SRC_DIR}/compress.c + ${LIBZ_SRC_DIR}/crc32.c + ${LIBZ_SRC_DIR}/deflate.c + ${LIBZ_SRC_DIR}/gzclose.c + ${LIBZ_SRC_DIR}/gzlib.c + ${LIBZ_SRC_DIR}/gzread.c + ${LIBZ_SRC_DIR}/gzwrite.c + ${LIBZ_SRC_DIR}/infback.c + ${LIBZ_SRC_DIR}/inffast.c + ${LIBZ_SRC_DIR}/inflate.c + ${LIBZ_SRC_DIR}/inftrees.c + ${LIBZ_SRC_DIR}/trees.c + ${LIBZ_SRC_DIR}/uncompr.c + ${LIBZ_SRC_DIR}/zutil.c +) + +set_target_properties(z_wasm PROPERTIES LINKER_LANGUAGE C) + +target_compile_definitions(z_wasm PRIVATE Z_HAVE_UNISTD_H _LARGEFILE64_SOURCE=1) + +target_compile_options(z_wasm + PRIVATE + -Wno-unused-function + -Wno-unused-variable +) + +target_include_directories(z_wasm + PUBLIC + ${LIBZ_SRC_DIR} +) + +################ BWA_WASM ################ +set(BWA_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +set(BWA_SOURCE + ${BWA_SRC_DIR}/utils.c + ${BWA_SRC_DIR}/kthread.c + ${BWA_SRC_DIR}/kstring.c + ${BWA_SRC_DIR}/ksw.c + ${BWA_SRC_DIR}/bwt.c + ${BWA_SRC_DIR}/bntseq.c + ${BWA_SRC_DIR}/bwa.c + ${BWA_SRC_DIR}/bwamem.c + ${BWA_SRC_DIR}/bwamem_pair.c + ${BWA_SRC_DIR}/bwamem_extra.c + ${BWA_SRC_DIR}/malloc_wrap.c + ${BWA_SRC_DIR}/QSufSort.c + ${BWA_SRC_DIR}/bwt_gen.c + ${BWA_SRC_DIR}/rope.c + ${BWA_SRC_DIR}/rle.c + ${BWA_SRC_DIR}/is.c + ${BWA_SRC_DIR}/bwtindex.c + ${BWA_SRC_DIR}/bwashm.c + ${BWA_SRC_DIR}/bwase.c + ${BWA_SRC_DIR}/bwaseqio.c + ${BWA_SRC_DIR}/bwtgap.c + ${BWA_SRC_DIR}/bwtaln.c + ${BWA_SRC_DIR}/bamlite.c + ${BWA_SRC_DIR}/bwape.c + ${BWA_SRC_DIR}/kopen.c + ${BWA_SRC_DIR}/pemerge.c + ${BWA_SRC_DIR}/maxk.c + ${BWA_SRC_DIR}/bwtsw2_core.c + ${BWA_SRC_DIR}/bwtsw2_main.c + ${BWA_SRC_DIR}/bwtsw2_aux.c + ${BWA_SRC_DIR}/bwt_lite.c + ${BWA_SRC_DIR}/bwtsw2_chain.c + ${BWA_SRC_DIR}/fastmap.c + ${BWA_SRC_DIR}/bwtsw2_pair.c + ${BWA_SRC_DIR}/main.c +) + +add_executable(${PROJECT_NAME} ${BWA_SOURCE}) + +set_target_properties(${PROJECT_NAME} PROPERTIES OUTPUT_NAME bwa.wasm) + +target_compile_definitions(${PROJECT_NAME} + PRIVATE + USE_MALLOC_WRAPPERS + __SSE__ __SSE2__ __SSE4_1__ + _WASI_EMULATED_MMAN _WASI_EMULATED_SIGNAL _WASI_EMULATED_PROCESS_CLOCKS +) + +target_compile_options(${PROJECT_NAME} + PRIVATE + -Wno-unused-function + -Wno-unused-variable + -msimd128 +) + +target_link_options(${PROJECT_NAME} + PRIVATE + -Wno-unused-command-line-argument + LINKER:--allow-undefined,--export=__heap_base,--export=__data_end + LINKER:-z,stack-size=1048576 +) + +target_link_libraries(${PROJECT_NAME} z_wasm wasi-emulated-process-clocks) + +add_custom_target(bwa_wasm_opt ALL + COMMAND + ${Binaryen_WASM_OPT} -Oz --enable-simd -o bwa.opt.wasm bwa.wasm + BYPRODUCTS + ${CMAKE_CURRENT_BINARY_DIR}/bwa.opt.wasm + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_dependencies(bwa_wasm_opt ${PROJECT_NAME}) diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/bwa/CMakeLists.txt b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/bwa/CMakeLists.txt new file mode 100644 index 00000000..5db52a38 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/bwa/CMakeLists.txt @@ -0,0 +1,73 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project(bwa_wasm) + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../cmake) + +################ dependencies ################ +find_package(Python3 REQUIRED) +find_package(WASISDK 16.0 REQUIRED) +execute_process( + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/../../../test-tools/pick-up-emscripten-headers/collect_files.py --install ../include --loglevel=ERROR + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) + +####################################### +include(ExternalProject) + +################ libz ################ +ExternalProject_Add(libz_src + GIT_REPOSITORY https://github.com/madler/zlib.git + GIT_TAG 04f42ceca40f73e2978b50e93806c2a18c1281fc + GIT_PROGRESS ON + GIT_SHALLOW ON + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/libz + UPDATE_COMMAND "" + PATCH_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" +) + +################ bwa ################ +ExternalProject_Add(bwa + GIT_REPOSITORY https://github.com/lh3/bwa.git + GIT_TAG 139f68fc4c3747813783a488aef2adc86626b01b + GIT_PROGRESS ON + GIT_SHALLOW ON + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/bwa + DEPENDS libz_src + UPDATE_COMMAND git clean -ffdx && git checkout -- * + && ${CMAKE_COMMAND} -E echo "Copying pre-installed CMakeLists.txt" + && ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.bwa_wasm.txt CMakeLists.txt + && git apply ../bwa.patch + CONFIGURE_COMMAND ${CMAKE_COMMAND} + -DWASI_SDK_PREFIX=${WASISDK_HOME} + -DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN} + -DCMAKE_SYSROOT=${WASISDK_SYSROOT} + -DCMAKE_C_FLAGS=-isystem\ ${CMAKE_CURRENT_SOURCE_DIR}/../include/sse\ -isystem\ ${CMAKE_CURRENT_SOURCE_DIR}/../include/libc/musl + ${CMAKE_CURRENT_SOURCE_DIR}/bwa + BUILD_COMMAND make bwa_wasm_opt -j 4 + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different ./bwa.opt.wasm ${CMAKE_CURRENT_BINARY_DIR}/bwa.wasm +) + +################ bwa data ################ +ExternalProject_Add(bwa-kit + PREFIX bwa-kit + URL https://sourceforge.net/projects/bio-bwa/files/bwakit/bwakit-0.7.15_x64-linux.tar.bz2/download + URL_HASH SHA256=0a7b11971bc7916b68e9df35a364afe77cb3000df02ffb3a6fbd1aff9be5878c + DOWNLOAD_NAME bwakit-0.7.15_x64-linux.tar.bz2 + DOWNLOAD_EXTRACT_TIMESTAMP ON + DOWNLOAD_NO_EXTRACT OFF + DOWNLOAD_NO_PROGRESS ON + UPDATE_COMMAND "" + PATCH_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${CMAKE_CURRENT_BINARY_DIR}/bwa-kit/src/bwa-kit/resource-GRCh38/hs38DH-extra.fa + ${CMAKE_CURRENT_BINARY_DIR}/hs38DH-extra.fa +) diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/bwa/README.md b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/bwa/README.md new file mode 100644 index 00000000..a8fbe3e6 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/bwa/README.md @@ -0,0 +1,46 @@ +"bwa" sample introduction +============== + +This sample demonstrates how to build [bwa](https://github.com/lh3/bwa) into +WebAssembly with simd support and run it with iwasm. + +## Preparation + +please refer to [installation instructions](../README.md). + +## Build + +``` shell +$ mkdir build && cd build +$ cmake .. +$ make +# to verify +$ ls bwa.wasm +``` + +## Download sample data + +Download the bwa-0.7.15 binary package from +[such an address](https://sourceforge.net/projects/bio-bwa/files/bwakit/bwakit-0.7.15_x64-linux.tar.bz2/download), +a sample data file named **hs38DH.fa** will be used later. + +If want more data, please refer to http://hgdownload.cse.ucsc.edu/goldenpath/hg19/bigZips/ + +## Run workload + +Firstly please build iwasm with simd support: + +``` shell +$ cd <wamr dir>/product-mini/platforms/linux/ +$ mkdir build && cd build +$ cmake .. +$ make +``` + +Then compile wasm file to aot file and run: + +``` shell +$ cd <wamr dir>/samples/workload/bwa/build +$ <wamr dir>/wamr-compiler/build/wamrc -o bwa.aot bwa.wasm +$ <wamr dir>/product-mini/platforms/linux/iwasm --dir=. bwa.aot index hs38DH.fa +``` diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/bwa/bwa.patch b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/bwa/bwa.patch new file mode 100644 index 00000000..5599ca31 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/bwa/bwa.patch @@ -0,0 +1,13 @@ +diff --git a/utils.c b/utils.c +index 9ceb1be..323299f 100644 +--- a/utils.c ++++ b/utils.c +@@ -301,6 +301,7 @@ long peakrss(void) + #ifdef __linux__ + return r.ru_maxrss * 1024; + #else +- return r.ru_maxrss; ++ /*return r.ru_maxrss;*/ ++ return 0; + #endif + } diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/cmake/FindBinaryen.cmake b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/cmake/FindBinaryen.cmake new file mode 100644 index 00000000..b4a64786 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/cmake/FindBinaryen.cmake @@ -0,0 +1,43 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# +# Output below variables: +# - Binaryen_HOME. the installation location +# + +include(CMakePrintHelpers) +include(FindPackageHandleStandardArgs) + +file(GLOB Binaryen_SEARCH_PATH "/opt/binaryen*") +find_path(Binaryen_HOME + NAMES bin/wasm-opt + PATHS ${Binaryen_SEARCH_PATH} + NO_CMAKE_FIND_ROOT_PATH + NO_SYSTEM_ENVIRONMENT_PATH + REQUIRED +) + +execute_process( + COMMAND ${Binaryen_HOME}/bin/wasm-opt --version + OUTPUT_VARIABLE WASM_OPT_OUTPUT + OUTPUT_STRIP_TRAILING_WHITESPACE +) + +string(REGEX MATCH version_[0-9]+ Binaryen_VERSION_tmp ${WASM_OPT_OUTPUT}) +string(REGEX MATCH [0-9]+ Binaryen_VERSION ${Binaryen_VERSION_tmp}) + +#cmake_print_variables(Binaryen_VERSION_tmp Binaryen_VERSION) + +find_package_handle_standard_args(Binaryen REQUIRED_VARS Binaryen_HOME VERSION_VAR Binaryen_VERSION) + +if(Binaryen_FOUND) + mark_as_advanced(Binaryen_SEARCH_PATH) + mark_as_advanced(Binaryen_VERSION_tmp) + mark_as_advanced(Binaryen_VERSION) + mark_as_advanced(WASM_OPT_OUTPUT) + + set(Binaryen_WASM_OPT ${Binaryen_HOME}/bin/wasm-opt) +else() + # TODO: install WASISDK +endif() diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/cmake/FindWASISDK.cmake b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/cmake/FindWASISDK.cmake new file mode 100644 index 00000000..fff8aea4 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/cmake/FindWASISDK.cmake @@ -0,0 +1,38 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# +# Output below variables: +# - WASISDK_HOME. the installation location +# - WASISDK_SYSROOT. where wasi-sysroot is +# - WASISDK_TOOLCHAIN. where wasi-sdk.cmake is +# + +include(CMakePrintHelpers) +include(FindPackageHandleStandardArgs) + +file(GLOB WASISDK_SEARCH_PATH "/opt/wasi-sdk-*") +find_path(WASISDK_HOME + NAMES share/wasi-sysroot + PATHS ${WASISDK_SEARCH_PATH} + NO_CMAKE_FIND_ROOT_PATH + NO_SYSTEM_ENVIRONMENT_PATH + REQUIRED +) + +string(REGEX MATCH [0-9]+\.[0-9]+\.*[0-9]* WASISDK_VERSION ${WASISDK_HOME}) + +#cmake_print_variables(WASISDK_HOME WASISDK_VERSION) +find_package_handle_standard_args(WASISDK REQUIRED_VARS WASISDK_HOME VERSION_VAR WASISDK_VERSION) + +if(WASISDK_FOUND) + mark_as_advanced(WASISDK_SEARCH_PATH) + mark_as_advanced(WASISDK_VERSION) + + set(WASISDK_CC_COMMAND ${WASISDK_HOME}/bin/clang) + set(WASISDK_CXX_COMMAND ${WASISDK_HOME}/bin/clang++) + set(WASISDK_SYSROOT ${WASISDK_HOME}/share/wasi-sysroot) + set(WASISDK_TOOLCHAIN ${WASISDK_HOME}/share/cmake/wasi-sdk.cmake) +else() + # TODO: install WASISDK +endif() diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/include/.gitkeep b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/include/.gitkeep new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/include/.gitkeep diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/meshoptimizer/.gitignore b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/meshoptimizer/.gitignore new file mode 100644 index 00000000..dd97754d --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/meshoptimizer/.gitignore @@ -0,0 +1,2 @@ +build +meshoptimizer
\ No newline at end of file diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/meshoptimizer/CMakeLists.txt b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/meshoptimizer/CMakeLists.txt new file mode 100644 index 00000000..d6a1c358 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/meshoptimizer/CMakeLists.txt @@ -0,0 +1,38 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project(bench-meshoptimizer) + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../cmake) + +################ dependencies ################ +find_package(Python3 REQUIRED) +find_package(WASISDK 16.0 REQUIRED) +execute_process( + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/../../../test-tools/pick-up-emscripten-headers/collect_files.py --install ../include --loglevel=ERROR + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) + +################ MESHOPTIMIZER ################ +include(ExternalProject) + +ExternalProject_Add(codecbench + PREFIX codecbench + GIT_REPOSITORY https://github.com/zeux/meshoptimizer.git + GIT_TAG f734fd572aed5bf76e84d9ed62ca6f4f6c47d84e + GIT_SHALLOW OFF + GIT_PROGRESS ON + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/meshoptimizer + UPDATE_COMMAND git clean -fd && git checkout -- * + && ${CMAKE_COMMAND} -E echo "Applying patch" + && git apply ${CMAKE_CURRENT_SOURCE_DIR}/codecbench.patch + CONFIGURE_COMMAND ${CMAKE_COMMAND} + -DWASI_SDK_PREFIX=${WASISDK_HOME} + -DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN} + -DCMAKE_SYSROOT=${WASISDK_SYSROOT} + ${CMAKE_CURRENT_SOURCE_DIR}/meshoptimizer + BUILD_COMMAND make codecbench -j 4 + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different ./codecbench.wasm ${CMAKE_CURRENT_BINARY_DIR}/codecbench.wasm +) diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/meshoptimizer/README.md b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/meshoptimizer/README.md new file mode 100644 index 00000000..466cd875 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/meshoptimizer/README.md @@ -0,0 +1,57 @@ +"codecbench of meshoptimizer" sample introduction +============== + +This sample demonstrates how to build [codecbench of messoptimizer](https://github.com/zeux/meshoptimizer) into +WebAssembly with simd support and run it with iwasm. + +## Preparation + +please refer to [installation instructions](../README.md). + +## Build with wasi-sdk + +``` shell +$ mkdir build && cd build +$ cmake .. +$ make +# to verify +$ ls codecbench.wasm +``` + +## Or build with EMSDK + +EMSDK is another toolchain to compile C/C++ code to WASM. In this case, the output wasm file +might have a higher performance than the file generated by wasi-sdk. + +``` shell +$ git clone https://github.com/zeux/meshoptimizer.git +$ cd messoptimizer +$ em++ tools/codecbench.cpp src/vertexcodec.cpp src/vertexfilter.cpp \ + src/overdrawanalyzer.cpp src/indexgenerator.cpp src/vcacheoptimizer.cpp \ + src/clusterizer.cpp src/indexcodec.cpp src/vfetchanalyzer.cpp \ + src/spatialorder.cpp src/allocator.cpp src/vcacheanalyzer.cpp \ + src/vfetchoptimizer.cpp src/overdrawoptimizer.cpp src/simplifier.cpp \ + src/stripifier.cpp -O3 -msimd128 \ + -s TOTAL_MEMORY=268435456 \ + -o codecbench.wasm +$ ls -l codecbench.wasm +``` + +## Run workload + +Firstly please build iwasm with simd support: + +``` shell +$ cd <wamr dir>/product-mini/platforms/linux/ +$ mkdir build && cd build +$ cmake .. +$ make +``` + +Then compile wasm file to aot file and run: + +``` shell +$ <wamr dir>/wamr-compiler/build/wamrc -o codecbench.aot codecbench.wasm +$ <wamr dir>/product-mini/platforms/linux/build/iwasm codecbench.aot +``` + diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/meshoptimizer/codecbench.patch b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/meshoptimizer/codecbench.patch new file mode 100644 index 00000000..19db792b --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/meshoptimizer/codecbench.patch @@ -0,0 +1,119 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 612cf3b..22a365a 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -158,3 +158,43 @@ install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/meshoptimizerConfigVersion.cmake + COMPONENT meshoptimizer + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/meshoptimizer) ++ ++################################################## ++# codecbench ++################################################## ++add_executable(codecbench tools/codecbench.cpp ${SOURCES}) ++ ++set_target_properties(codecbench PROPERTIES OUTPUT_NAME codecbench.wasm) ++ ++target_compile_options(codecbench ++ PUBLIC ++ -O3 -msimd128 ++ -std=c++11 ++ -Wno-unused-function ++ -Wno-unused-variable ++) ++ ++target_link_options(codecbench ++ PUBLIC ++ LINKER:-allow-undefined,--demangle,--export=malloc,--export=free ++) ++ ++find_program(WASM_OPT ++ NAMES wasm-opt ++ PATHS /opt/binaryen-version_97/bin /opt/binaryen/bin ++) ++ ++if (NOT WASM_OPT) ++ message(WARNING "can not find wasm-opt and will not optimize any wasm module") ++endif() ++ ++add_custom_target(codecbench.opt ALL ++ COMMAND ++ ${WASM_OPT} -Oz --enable-simd -o codecbench.opt.wasm codecbench.wasm ++ BYPRODUCTS ++ ${CMAKE_CURRENT_BINARY_DIR}/codecbench.opt.wasm ++ WORKING_DIRECTORY ++ ${CMAKE_CURRENT_BINARY_DIR} ++) ++ ++add_dependencies(codecbench.opt codecbench) +diff --git a/src/vertexcodec.cpp b/src/vertexcodec.cpp +index 4bd1112..257c258 100644 +--- a/src/vertexcodec.cpp ++++ b/src/vertexcodec.cpp +@@ -89,13 +89,13 @@ + #endif + + #ifdef SIMD_WASM +-#define wasmx_splat_v32x4(v, i) wasm_v32x4_shuffle(v, v, i, i, i, i) +-#define wasmx_unpacklo_v8x16(a, b) wasm_v8x16_shuffle(a, b, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23) +-#define wasmx_unpackhi_v8x16(a, b) wasm_v8x16_shuffle(a, b, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31) +-#define wasmx_unpacklo_v16x8(a, b) wasm_v16x8_shuffle(a, b, 0, 8, 1, 9, 2, 10, 3, 11) +-#define wasmx_unpackhi_v16x8(a, b) wasm_v16x8_shuffle(a, b, 4, 12, 5, 13, 6, 14, 7, 15) +-#define wasmx_unpacklo_v64x2(a, b) wasm_v64x2_shuffle(a, b, 0, 2) +-#define wasmx_unpackhi_v64x2(a, b) wasm_v64x2_shuffle(a, b, 1, 3) ++#define wasmx_splat_v32x4(v, i) wasm_i32x4_shuffle(v, v, i, i, i, i) ++#define wasmx_unpacklo_v8x16(a, b) wasm_i8x16_shuffle(a, b, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23) ++#define wasmx_unpackhi_v8x16(a, b) wasm_i8x16_shuffle(a, b, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31) ++#define wasmx_unpacklo_v16x8(a, b) wasm_i16x8_shuffle(a, b, 0, 8, 1, 9, 2, 10, 3, 11) ++#define wasmx_unpackhi_v16x8(a, b) wasm_i16x8_shuffle(a, b, 4, 12, 5, 13, 6, 14, 7, 15) ++#define wasmx_unpacklo_v64x2(a, b) wasm_i64x2_shuffle(a, b, 0, 2) ++#define wasmx_unpackhi_v64x2(a, b) wasm_i64x2_shuffle(a, b, 1, 3) + #endif + + namespace meshopt +@@ -757,7 +757,7 @@ static v128_t decodeShuffleMask(unsigned char mask0, unsigned char mask1) + v128_t sm1 = wasm_v128_load(&kDecodeBytesGroupShuffle[mask1]); + + v128_t sm1off = wasm_v128_load(&kDecodeBytesGroupCount[mask0]); +- sm1off = wasm_v8x16_shuffle(sm1off, sm1off, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); ++ sm1off = wasm_i8x16_shuffle(sm1off, sm1off, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + + v128_t sm1r = wasm_i8x16_add(sm1, sm1off); + +@@ -807,7 +807,7 @@ static const unsigned char* decodeBytesGroupSimd(const unsigned char* data, unsi + + v128_t shuf = decodeShuffleMask(mask0, mask1); + +- v128_t result = wasm_v128_bitselect(wasm_v8x16_swizzle(rest, shuf), sel, mask); ++ v128_t result = wasm_v128_bitselect(wasm_i8x16_swizzle(rest, shuf), sel, mask); + + wasm_v128_store(buffer, result); + +@@ -829,7 +829,7 @@ static const unsigned char* decodeBytesGroupSimd(const unsigned char* data, unsi + + v128_t shuf = decodeShuffleMask(mask0, mask1); + +- v128_t result = wasm_v128_bitselect(wasm_v8x16_swizzle(rest, shuf), sel, mask); ++ v128_t result = wasm_v128_bitselect(wasm_i8x16_swizzle(rest, shuf), sel, mask); + + wasm_v128_store(buffer, result); + +diff --git a/src/vertexfilter.cpp b/src/vertexfilter.cpp +index 5c7589c..c79cad4 100644 +--- a/src/vertexfilter.cpp ++++ b/src/vertexfilter.cpp +@@ -57,10 +57,10 @@ + #endif + + #ifdef SIMD_WASM +-#define wasmx_unpacklo_v16x8(a, b) wasm_v16x8_shuffle(a, b, 0, 8, 1, 9, 2, 10, 3, 11) +-#define wasmx_unpackhi_v16x8(a, b) wasm_v16x8_shuffle(a, b, 4, 12, 5, 13, 6, 14, 7, 15) +-#define wasmx_unziplo_v32x4(a, b) wasm_v32x4_shuffle(a, b, 0, 2, 4, 6) +-#define wasmx_unziphi_v32x4(a, b) wasm_v32x4_shuffle(a, b, 1, 3, 5, 7) ++#define wasmx_unpacklo_v16x8(a, b) wasm_i16x8_shuffle(a, b, 0, 8, 1, 9, 2, 10, 3, 11) ++#define wasmx_unpackhi_v16x8(a, b) wasm_i16x8_shuffle(a, b, 4, 12, 5, 13, 6, 14, 7, 15) ++#define wasmx_unziplo_v32x4(a, b) wasm_i32x4_shuffle(a, b, 0, 2, 4, 6) ++#define wasmx_unziphi_v32x4(a, b) wasm_i32x4_shuffle(a, b, 1, 3, 5, 7) + #endif + + #ifndef __has_builtin diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/preparation.sh b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/preparation.sh new file mode 100755 index 00000000..47b11ac5 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/preparation.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +readonly BUILD_CONTENT="/tmp/build_content" +readonly WABT_VER=1.0.31 +readonly WABT_FILE="wabt-${WABT_VER}-ubuntu.tar.gz" +readonly CMAKE_VER=3.25.1 +readonly CMAKE_FILE="cmake-${CMAKE_VER}-Linux-x86_64.sh" +readonly BINARYEN_VER=version_111 +readonly BINARYEN_FILE="binaryen-${BINARYEN_VER}-x86_64-linux.tar.gz" +readonly BAZEL_VER=6.0.0 +readonly BAZEL_FILE=bazel-${BAZEL_VER}-installer-linux-x86_64.sh + +function DEBUG() { + env | grep -q "\<DEBUG\>" +} + +# +# install dependency +function install_deps() { + apt update + apt install -y lsb-release wget software-properties-common \ + build-essential git tree zip unzip +} + +# +# install wabt +function install_wabt() { + if [[ ! -f ${WABT_FILE} ]]; then + wget https://github.com/WebAssembly/wabt/releases/download/${WABT_VER}/${WABT_FILE} + fi + + tar zxf ${WABT_FILE} -C /opt + ln -sf /opt/wabt-${WABT_VER} /opt/wabt +} + +# +# install cmake +function install_cmake() { + if [[ ! -f cmake-${CMAKE_VER}-Linux-x86_64.sh ]]; then + wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VER}/${CMAKE_FILE} + fi + + chmod a+x ${CMAKE_FILE} + mkdir /opt/cmake + ./${CMAKE_FILE} --prefix=/opt/cmake --skip-license + ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake +} + +# +# install emsdk +function install_emsdk() { + cd /opt + git clone https://github.com/emscripten-core/emsdk.git + cd emsdk + git pull + ./emsdk install 3.1.28 + ./emsdk activate 3.1.28 + echo "source /opt/emsdk/emsdk_env.sh" >> "${HOME}"/.bashrc +} + +# +# install binaryen +function install_binaryen() { + if [[ ! -f ${BINARYEN_FILE} ]]; then + wget https://github.com/WebAssembly/binaryen/releases/download/${BINARYEN_VER}/${BINARYEN_FILE} + fi + + tar zxf ${BINARYEN_FILE} -C /opt + ln -sf /opt/binaryen-${BINARYEN_VER} /opt/binaryen +} + +# +# install bazel +function install_bazel() { + if [[ ! -f ${BAZEL_FILE} ]]; then + wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VER}/${BAZEL_FILE} + fi + + chmod a+x ${BAZEL_FILE} + ./${BAZEL_FILE} +} + +# +# MAIN +DEBUG && set -xevu +if [[ ! -d ${BUILD_CONTENT} ]]; then + mkdir ${BUILD_CONTENT} +fi + +cd ${BUILD_CONTENT} || exit +if DEBUG; then + "$@" +else + install_deps \ + && install_bazel \ + && install_binaryen \ + && install_cmake \ + && install_emsdk \ + && install_wabt \ + && install_wasi-sdk +fi +cd - > /dev/null || exit +DEBUG && set +xevu diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/tensorflow/README.md b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/tensorflow/README.md new file mode 100644 index 00000000..7bc7dd25 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/tensorflow/README.md @@ -0,0 +1,19 @@ +"tensorflow" sample introduction +============== + +This sample demonstrates how to build [tensorflow](https://github.com/tensorflow/tensorflow) into WebAssembly with emsdk toolchain and run it with iwasm.: +```bash +./build.sh +# for linux platform, or +./build.sh --threads +# for multi-threading on linux platform +./build.sh --sgx +# for linux-sgx platform +``` +to build tensorflow and run it with iwasm, which basically contains the following steps: +- clone emsdk under `<wamr_dir>/core/deps`, install and activate 2.0.26 +- hack emcc to delete some objects in libc.a +- build tf-lite with emcc compiler +- build iwasm with lib-pthread and libc-emcc enabled +- run benchmark model with iwasm: + --max-secs 300: means the max training time cost is 5 minutes, you can adjust it by yourself diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/tensorflow/build.sh b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/tensorflow/build.sh new file mode 100755 index 00000000..6df8db42 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/tensorflow/build.sh @@ -0,0 +1,157 @@ +#!/bin/bash + +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +#################################### +# build tensorflow-lite sample # +#################################### +BUILD_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +WAMR_DIR="${BUILD_SCRIPT_DIR}/../../.." +WAMR_PLATFORM_DIR="${WAMR_DIR}/product-mini/platforms" +WAMRC_DIR="${WAMR_DIR}/wamr-compiler" +CORE_DEPS_DIR="${WAMR_DIR}/core/deps" +EMSDK_DIR="${CORE_DEPS_DIR}/emsdk" + +EMSDK_WASM_DIR="${EMSDK_DIR}/upstream/emscripten/cache/sysroot/lib/wasm32-emscripten" +OUT_DIR="${BUILD_SCRIPT_DIR}/out" +TENSORFLOW_DIR="${BUILD_SCRIPT_DIR}/tensorflow" +TF_LITE_BUILD_DIR="${TENSORFLOW_DIR}/tensorflow/lite/tools/make" + +function Clear_Before_Exit() +{ + [[ -f ${TENSORFLOW_DIR}/tf_lite.patch ]] && + rm -f ${TENSORFLOW_DIR}/tf_lite.patch + # resume the libc.a under EMSDK_WASM_DIR + cd ${EMSDK_WASM_DIR} + mv libc.a.bak libc.a +} + +set -xe + +# 1.clone emsdk +cd ${CORE_DEPS_DIR} +rm -fr emsdk +git clone https://github.com/emscripten-core/emsdk.git +cd emsdk +./emsdk install 2.0.26 +./emsdk activate 2.0.26 +source emsdk_env.sh + +# 2.hack emcc +cd ${EMSDK_WASM_DIR} +# back up libc.a +cp libc.a libc.a.bak +# delete some objects in libc.a +emar d libc.a open.o +emar d libc.a mmap.o +emar d libc.a munmap.o +emar d libc.a library_pthread_stub.o +emar d libc.a pthread_self.o +emranlib libc.a + +# 3. build tf-lite +cd ${BUILD_SCRIPT_DIR} +# 3.1 clone tf repo from Github and checkout to 2303ed commit +if [ ! -d "tensorflow" ]; then + git clone https://github.com/tensorflow/tensorflow.git +fi + +cd ${TENSORFLOW_DIR} +git checkout 2303ed4bdb344a1fc4545658d1df6d9ce20331dd + +# 3.2 copy the tf-lite.patch to tensorflow_root_dir and apply it +cd ${TENSORFLOW_DIR} +cp ${BUILD_SCRIPT_DIR}/tf_lite.patch . +git checkout tensorflow/lite/tools/make/Makefile +git checkout tensorflow/lite/tools/make/targets/linux_makefile.inc + +if [[ $(git apply tf_lite.patch 2>&1) =~ "error" ]]; then + echo "git apply patch failed, please check tf-lite related changes..." + Clear_Before_Exit + exit 0 +fi + +cd ${TF_LITE_BUILD_DIR} +# 3.3 download dependencies +if [ ! -d "${TF_LITE_BUILD_DIR}/downloads" ]; then + source download_dependencies.sh +fi + +# 3.4 build tf-lite target +if [ -d "${TF_LITE_BUILD_DIR}/gen" ]; then + rm -fr ${TF_LITE_BUILD_DIR}/gen +fi + +make -j 4 -C "${TENSORFLOW_DIR}" -f ${TF_LITE_BUILD_DIR}/Makefile + +# remove patch file and recover emcc libc.a after building +Clear_Before_Exit + +# 3.5 copy /make/gen target files to out/ +rm -rf ${OUT_DIR} +mkdir ${OUT_DIR} +cp -r ${TF_LITE_BUILD_DIR}/gen/linux_x86_64/bin/. ${OUT_DIR}/ + +# 4. compile tf-model.wasm to tf-model.aot with wamrc +# 4.1 build wamr-compiler +cd ${WAMRC_DIR} +./build_llvm.sh +rm -fr build && mkdir build +cd build && cmake .. +make +# 4.2 compile tf-mode.wasm to tf-model.aot +WAMRC_CMD="$(pwd)/wamrc" +cd ${OUT_DIR} +if [[ $1 == '--sgx' ]]; then + ${WAMRC_CMD} -sgx -o benchmark_model.aot benchmark_model.wasm +elif [[ $1 == '--threads' ]]; then + ${WAMRC_CMD} --enable-multi-thread -o benchmark_model.aot benchmark_model.wasm +else + ${WAMRC_CMD} -o benchmark_model.aot benchmark_model.wasm +fi + +# 5. build iwasm with pthread and libc_emcc enable +# platform: +# linux by default +# linux-sgx if $1 equals '--sgx' +if [[ $1 == '--sgx' ]]; then + cd ${WAMR_PLATFORM_DIR}/linux-sgx + rm -fr build && mkdir build + cd build && cmake .. -DWAMR_BUILD_LIBC_EMCC=1 + make + cd ../enclave-sample + make +else + cd ${WAMR_PLATFORM_DIR}/linux + rm -fr build && mkdir build + cd build && cmake .. -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIBC_EMCC=1 + make +fi + +# 6. run tensorflow with iwasm +cd ${OUT_DIR} +# 6.1 download tf-lite model +wget "https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip" +unzip mobilenet_v1_224_android_quant_2017_11_08.zip + +# 6.2 run tf-lite model with iwasm +echo "---> run tensorflow benchmark model with iwasm" +if [[ $1 == '--sgx' ]]; then + IWASM_CMD="${WAMR_PLATFORM_DIR}/linux-sgx/enclave-sample/iwasm" +else + IWASM_CMD="${WAMR_PLATFORM_DIR}/linux/build/iwasm" +fi + +if [[ $1 == '--threads' ]]; then + ${IWASM_CMD} --heap-size=10475860 \ + benchmark_model.aot --num_threads=4 \ + --graph=mobilenet_quant_v1_224.tflite --max_secs=300 +else + ${IWASM_CMD} --heap-size=10475860 \ + benchmark_model.aot \ + --graph=mobilenet_quant_v1_224.tflite --max_secs=300 +fi diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/tensorflow/tf_lite.patch b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/tensorflow/tf_lite.patch new file mode 100644 index 00000000..b6224d58 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/tensorflow/tf_lite.patch @@ -0,0 +1,84 @@ +diff --git a/tensorflow/lite/tools/make/Makefile b/tensorflow/lite/tools/make/Makefile +index c7ddff58440..ebfebaead35 100644 +--- a/tensorflow/lite/tools/make/Makefile ++++ b/tensorflow/lite/tools/make/Makefile +@@ -48,11 +48,7 @@ INCLUDES += -I/usr/local/include + + # These are the default libraries needed, but they can be added to or + # overridden by the platform-specific settings in target makefiles. +-LIBS := \ +--lstdc++ \ +--lpthread \ +--lm \ +--lz \ ++LIBS := -lm \ + -ldl + + # There are no rules for compiling objects for the host system (since we don't +@@ -84,14 +80,24 @@ endif # ifeq ($(HOST_ARCH),$(TARGET_ARCH)) + endif # ifeq ($(HOST_OS),$(TARGET)) + endif + ++CFLAGS+=-msimd128 -mbulk-memory -matomics ++CXXFLAGS+=-msimd128 -mbulk-memory -matomics ++ ++LIBFLAGS += -s TOTAL_STACK=1048576 -s MALLOC="none" \ ++ -s INITIAL_MEMORY=16777216 \ ++ -s MAXIMUM_MEMORY=167772160 \ ++ -s ALLOW_MEMORY_GROWTH=1 \ ++ -Wl,--export=__data_end -Wl,--export=__heap_base,--shared-memory,--no-check-features \ ++ -s ERROR_ON_UNDEFINED_SYMBOLS=0 ++ + # This library is the main target for this makefile. It will contain a minimal + # runtime that can be linked in to other programs. + LIB_NAME := libtensorflow-lite.a + + # Benchmark static library and binary + BENCHMARK_LIB_NAME := benchmark-lib.a +-BENCHMARK_BINARY_NAME := benchmark_model +-BENCHMARK_PERF_OPTIONS_BINARY_NAME := benchmark_model_performance_options ++BENCHMARK_BINARY_NAME := benchmark_model.wasm ++BENCHMARK_PERF_OPTIONS_BINARY_NAME := benchmark_model_performance_options.wasm + + # A small example program that shows how to link against the library. + MINIMAL_SRCS := \ +@@ -277,12 +283,16 @@ LIB_PATH := $(LIBDIR)$(LIB_NAME) + BENCHMARK_LIB := $(LIBDIR)$(BENCHMARK_LIB_NAME) + BENCHMARK_BINARY := $(BINDIR)$(BENCHMARK_BINARY_NAME) + BENCHMARK_PERF_OPTIONS_BINARY := $(BINDIR)$(BENCHMARK_PERF_OPTIONS_BINARY_NAME) +-MINIMAL_BINARY := $(BINDIR)minimal ++MINIMAL_BINARY := $(BINDIR)minimal.wasm + LABEL_IMAGE_BINARY := $(BINDIR)label_image + +-CXX := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}g++ +-CC := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}gcc +-AR := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}ar ++# CXX := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}g++ ++# CC := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}gcc ++# AR := $(CC_PREFIX)${TARGET_TOOLCHAIN_PREFIX}ar ++ ++CXX := em++ ++CC := emcc ++AR := emar + + MINIMAL_OBJS := $(addprefix $(OBJDIR), \ + $(patsubst %.cc,%.o,$(patsubst %.c,%.o,$(MINIMAL_SRCS)))) +diff --git a/tensorflow/lite/tools/make/targets/linux_makefile.inc b/tensorflow/lite/tools/make/targets/linux_makefile.inc +index 222cef9e5ff..eea89a38f01 100644 +--- a/tensorflow/lite/tools/make/targets/linux_makefile.inc ++++ b/tensorflow/lite/tools/make/targets/linux_makefile.inc +@@ -2,12 +2,10 @@ + ifeq ($(TARGET), linux) + CXXFLAGS += \ + -fPIC \ +- -DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK \ +- -pthread ++ -DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK + CFLAGS += \ + -fPIC \ +- -DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK \ +- -pthread ++ -DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK + # TODO(petewarden): In the future we may want to add architecture-specific + # flags like -msse4.2 + LIBS += -ldl diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/.gitignore b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/.gitignore new file mode 100644 index 00000000..e8bec70e --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/.gitignore @@ -0,0 +1,8 @@ +# from CMakeLists +av1 +build +include + +# from build.sh +wasm-av1 +out diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/CMakeLists.avx_wasm.txt b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/CMakeLists.avx_wasm.txt new file mode 100644 index 00000000..409665b3 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/CMakeLists.avx_wasm.txt @@ -0,0 +1,72 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project(testavx) + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../cmake) + +################ dependencies ################ +find_package(Binaryen 111 REQUIRED) + +################ AOM ################ +set(ENABLE_CCACHE ON) +set(ENABLE_DOCS OFF CACHE BOOL "ENABLE_DOCS" FORCE) +set(ENABLE_EXAMPLES OFF CACHE BOOL "ENABLE_EXAMPLES" FORCE) +set(ENABLE_NEON OFF CACHE BOOL "ENABLE_EXAMPLES" FORCE) +set(ENABLE_NEON_ASM OFF CACHE BOOL "ENABLE_EXAMPLES" FORCE) +set(ENABLE_VSX OFF CACHE BOOL "ENABLE_EXAMPLES" FORCE) +set(ENABLE_MMX OFF CACHE BOOL "ENABLE_EXAMPLES" FORCE) +set(AOM_TARGET_CPU generic) +set(CONFIG_ACCOUNTING 1 CACHE NUMBER "" FORCE) +set(CONFIG_INSPECTION 1 CACHE NUMBER "" FORCE) +set(CONFIG_MULTITHREAD 0 CACHE NUMBER "" FORCE) +set(CONFIG_RUNTIME_CPU_DETECT 0 CACHE NUMBER "" FORCE) +set(CONFIG_UNIT_TESTS 0 CACHE NUMBER "" FORCE) +set(CONFIG_WEBM_IO 0 CACHE NUMBER "" FORCE) +add_subdirectory(third_party/aom third_party/aom/bin EXCLUDE_FROM_ALL) + +################ AV ################ +add_executable(${PROJECT_NAME} + test.c + decode-av1.c +) + +target_include_directories(${PROJECT_NAME} + PRIVATE + third_party/aom/ + ${CMAKE_CURRENT_BINARY_DIR}/third_party/aom/bin +) + +set_target_properties(${PROJECT_NAME} + PROPERTIES + OUTPUT_NAME ${PROJECT_NAME}.wasm +) + +target_link_options(${PROJECT_NAME} + PRIVATE + LINKER:--allow-undefined + LINKER:--export=__heap_base + LINKER:--export=__data_end + LINKER:--initial-memory=33554432 + LINKER:-z,stack-size=25165824 +) + +target_link_libraries(${PROJECT_NAME} + PRIVATE + aom +) + +add_dependencies(${PROJECT_NAME} aom) + +add_custom_target(${PROJECT_NAME}_opt ALL + COMMAND + ${Binaryen_WASM_OPT} -Oz --enable-simd -o ${PROJECT_NAME}.opt.wasm ${PROJECT_NAME}.wasm + BYPRODUCTS + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.opt.wasm + WORKING_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR} +) + +add_dependencies(${PROJECT_NAME}_opt ${PROJECT_NAME}) diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/CMakeLists.txt b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/CMakeLists.txt new file mode 100644 index 00000000..3d263bfb --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/CMakeLists.txt @@ -0,0 +1,44 @@ +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required (VERSION 3.14) + +project(av1_wasm) + +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../cmake) + +################ dependencies ################ +find_package(Python3 REQUIRED) +find_package(WASISDK 16.0 REQUIRED) +execute_process( + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/../../../test-tools/pick-up-emscripten-headers/collect_files.py --install ../include --loglevel=ERROR + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) + +####################################### +include(ExternalProject) + +################ av1 ################ +ExternalProject_Add(av1 + PREFIX av1 + GIT_REPOSITORY https://github.com/GoogleChromeLabs/wasm-av1.git + GIT_TAG master + GIT_PROGRESS ON + GIT_SHALLOW ON + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/av1 + UPDATE_COMMAND git clean -fd && git checkout -- * + && ${CMAKE_COMMAND} -E echo "Copying pre-installed CMakeLists.txt" + && ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.avx_wasm.txt CMakeLists.txt + && git apply ../av1-clang.patch + CONFIGURE_COMMAND ${CMAKE_COMMAND} + -DWASI_SDK_PREFIX=${WASISDK_HOME} + -DCMAKE_TOOLCHAIN_FILE=${WASISDK_TOOLCHAIN} + -DCMAKE_SYSROOT=${WASISDK_SYSROOT} + -DCMAKE_C_FLAGS=-isystem\ ${CMAKE_CURRENT_SOURCE_DIR}/../include/sse\ -isystem\ ${CMAKE_CURRENT_SOURCE_DIR}/../include/libc/musl + ${CMAKE_CURRENT_SOURCE_DIR}/av1 + BUILD_COMMAND make testavx_opt -j 4 + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different + testavx.opt.wasm + ${CMAKE_CURRENT_SOURCE_DIR}/av1/third_party/samples/elephants_dream_480p24.ivf + ${CMAKE_CURRENT_BINARY_DIR} +) diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/README.md b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/README.md new file mode 100644 index 00000000..2166fe6a --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/README.md @@ -0,0 +1,54 @@ +"wasm-av1" sample introduction +============== + +This sample demonstrates how to build [wasm-av1](https://github.com/GoogleChromeLabs/wasm-av1) into +WebAssembly with simd support and run it with iwasm. + +## Preparation + +please refer to [installation instructions](../README.md). + +## Build with wasi-sdk + +``` shell +$ mkdir build && cd build +$ cmake .. +$ make +# to verify +$ ls testavx.wasm +``` + +## Or build with EMSDK + +just run the convenience script: + +```bash +./build.sh +``` + +the script builds wasm-av1 and runs it with iwasm, which basically contains the following steps: +- hack emcc to delete some objects in libc.a +- patch wasm-av1 and build it with emcc compiler +- build iwasm with simd and libc-emcc support +- run testav1.aot with iwasm + +### Run workload + +Firstly please build iwasm with simd support: + +``` shell +$ cd <wamr dir>/product-mini/platforms/linux/ +$ mkdir build && cd build +$ cmake .. -DWAMR_BUILD_LIBC_EMCC=1 +$ make +``` + +Then compile wasm file to aot file and run: + +``` shell +$ cd <dir of testavx.wasm> +$ <wamr dir>/wamr-compiler/build/wamrc -o testavx.aot testavx.wasm +# copy sample data like <wamr dir>/samples/workload/wasm-av1/av1/third_party/samples/elephants_dream_480p24.ivf +# make sure you declare the access priority of the directory in which the sample data is +$ <wamr dir>/product-mini/platforms/linux/build/iwasm --dir=. testavx.aot elephants_dream_480p24.ivf +``` diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/av1-clang.patch b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/av1-clang.patch new file mode 100644 index 00000000..97e79548 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/av1-clang.patch @@ -0,0 +1,19 @@ +diff --git a/test.c b/test.c +index df2d44b..520bf13 100644 +--- a/test.c ++++ b/test.c +@@ -63,9 +63,14 @@ main(int argc, char *argv[]) { + static int i = 0; + + ++i; ++ printf("Decoding frame #%d\n", i); + if (30 <= i && i < 40) { ++ printf("Dumping frame #%d\n", i); + dump_raw_frame(af, i); + } ++ if (i >= 1000) { ++ break; ++ } + } + /* + * Run the decoder every time, so that we keep diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/build.sh b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/build.sh new file mode 100755 index 00000000..efa17eca --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/build.sh @@ -0,0 +1,100 @@ +# +# Copyright (C) 2019 Intel Corporation. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# + +#!/bin/bash + +#################################### +# build wasm-av1 sample # +#################################### +if [ ! -d "${EMSDK}" ]; then + echo "can not find emsdk. " + echo "please refer to https://emscripten.org/docs/getting_started/downloads.html " + echo "to install it, or active it by 'source <emsdk_dir>emsdk_env.sh'" + exit +fi + +set -xe + +EMSDK_WASM_DIR="${EMSDK}/upstream/emscripten/cache/sysroot/lib/wasm32-emscripten" +BUILD_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUT_DIR="${BUILD_SCRIPT_DIR}/out" +WASM_AV1_DIR="${BUILD_SCRIPT_DIR}/wasm-av1" + +WAMR_PLATFORM_DIR="${BUILD_SCRIPT_DIR}/../../../product-mini/platforms" +IWASM_CMD="${WAMR_PLATFORM_DIR}/linux/build/iwasm" + +WAMRC_DIR="${BUILD_SCRIPT_DIR}/../../../wamr-compiler" +WAMRC_CMD="${WAMRC_DIR}/build/wamrc" + +function Clear_Before_Exit +{ + [[ -f ${WASM_AV1_DIR}/wasm-av1.patch ]] && + rm -f ${WASM_AV1_DIR}/wasm-av1.patch + # resume the libc.a under EMSDK_WASM_DIR + cd ${EMSDK_WASM_DIR} + mv libc.a.bak libc.a +} + +# 1.hack emcc +cd ${EMSDK_WASM_DIR} +# back up libc.a +cp libc.a libc.a.bak +# delete some objects in libc.a +emar d libc.a fopen.o +emar d libc.a fread.o +emar d libc.a feof.o +emar d libc.a fclose.o + +# 2. build wasm-av1 +cd ${BUILD_SCRIPT_DIR} +# 2.1 clone wasm-av1 repo from Github +if [ ! -d "wasm-av1" ]; then + git clone https://github.com/GoogleChromeLabs/wasm-av1.git +fi + +# 2.2 copy the wasm-av1.patch to wasm-av1 and apply the patch +cd ${WASM_AV1_DIR} +cp -a ${BUILD_SCRIPT_DIR}/wasm-av1.patch . +git checkout Makefile +git checkout test.c +git checkout third_party/aom + +if [[ $(git apply wasm-av1.patch 2>&1) =~ "error" ]]; then + echo "git apply patch failed, please check wasm-av1 related changes..." + Clear_Before_Exit + exit 0 +fi + +make testavx -j 4 + +# remove patch file and recover emcc libc.a after building +Clear_Before_Exit + +# 2.3 copy /make/gen target files to out/ +rm -rf ${OUT_DIR} && mkdir ${OUT_DIR} +cp -a ${WASM_AV1_DIR}/testavx.wasm ${OUT_DIR}/ + +# 3. compile wasm-av1.wasm to wasm-av1.aot with wamrc +# 3.1 build wamr-compiler +cd ${WAMRC_DIR} +./build_llvm.sh +rm -fr build && mkdir build +cd build && cmake .. +make +# 3.2 compile wasm-av1.wasm to wasm-av1.aot +cd ${OUT_DIR} +${WAMRC_CMD} -o testavx.aot testavx.wasm + +# 4. build iwasm with pthread and libc_emcc enable +cd ${WAMR_PLATFORM_DIR}/linux +rm -fr build && mkdir build +cd build && cmake .. -DWAMR_BUILD_LIB_PTHREAD=1 -DWAMR_BUILD_LIBC_EMCC=1 +make + +# 5. run wasm-av1 with iwasm +echo "---> run testav1.aot with iwasm" +cd ${OUT_DIR} +${IWASM_CMD} testavx.aot ../wasm-av1/third_party/samples/elephants_dream_480p24.ivf + diff --git a/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/wasm-av1.patch b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/wasm-av1.patch new file mode 100644 index 00000000..98f26256 --- /dev/null +++ b/fluent-bit/lib/wasm-micro-runtime-WAMR-1.2.2/samples/workload/wasm-av1/wasm-av1.patch @@ -0,0 +1,701 @@ +diff --git a/Makefile b/Makefile +index c39fff6..4682d43 100644 +--- a/Makefile ++++ b/Makefile +@@ -59,11 +59,13 @@ $(TARGET): $(DEPS) blob-api.c yuv-to-rgb.c $(EMLIBAV1) + ]" \ + blob-api.c yuv-to-rgb.c $(SRCS) $(INC) -L $(LIBDIR) -l$(LIB) + +-$(TESTTARGET): test.c $(DEPS) $(X86LIBAV1) +- cc -o $@ -O3 test.c $(SRCS) $(INC) -L $(X86LIBDIR) -l$(LIB) ++$(TESTTARGET): test.c $(DEPS) $(EMLIBAV1) ++ emcc -o $@.wasm -O3 test.c $(SRCS) $(INC) -L $(LIBDIR) -l$(LIB) \ ++ -s TOTAL_MEMORY=104857600 -s ERROR_ON_UNDEFINED_SYMBOLS=0 + +-$(TESTTARGET)g: test.c $(DEPS) $(X86LIBAV1) +- cc -o $@ -g test.c $(SRCS) $(INC) -L $(X86LIBDIR) -l$(LIB) ++$(TESTTARGET)g: test.c $(DEPS) $(EMLIBAV1) ++ emcc -o $@.wasm -g test.c $(SRCS) $(INC) -L $(LIBDIR) -l$(LIB) \ ++ -s TOTAL_MEMORY=104857600 -s ERROR_ON_UNDEFINED_SYMBOLS=0 + + clean: + -rm $(TARGET) $(TESTTARGET) $(TESTTARGET)g +@@ -80,7 +82,7 @@ $(EMLIBAV1): $(LIBDIR) + -DCONFIG_RUNTIME_CPU_DETECT=0 \ + -DCONFIG_UNIT_TESTS=0 \ + -DCONFIG_WEBM_IO=0 \ +- -DCMAKE_TOOLCHAIN_FILE=`../../get-emcmake.sh`; \ ++ -DCMAKE_TOOLCHAIN_FILE=${EMSDK}/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake; \ + make \ + ) + +diff --git a/test.c b/test.c +index df2d44b..cb270de 100644 +--- a/test.c ++++ b/test.c +@@ -18,6 +18,9 @@ + + #include "decode-av1-priv.h" + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + static void + dump_raw_frame(AVX_Video_Frame *avf, int id) { + FILE *f; +@@ -26,12 +29,13 @@ dump_raw_frame(AVX_Video_Frame *avf, int id) { + void *buf; + + sprintf(name, "frame%04d.yuv", id); ++ printf("writing %s ..\n", name); + if ((f = fopen(name, "wb")) == NULL) { + return; + } + buf = AVX_Video_Frame_get_buffer(avf); + size = AVX_Video_Frame_get_size(avf); +- fwrite(buf, size, 1, f); ++ emcc_fwrite(buf, size, 1, f); + fclose(f); + } + +@@ -63,9 +67,12 @@ main(int argc, char *argv[]) { + static int i = 0; + + ++i; ++ printf("##decode raw frame %d\n", i); + if (30 <= i && i < 40) { + dump_raw_frame(af, i); + } ++ if (i >= 1000) ++ break; + } + /* + * Run the decoder every time, so that we keep +diff --git a/third_party/aom/CMakeLists.txt b/third_party/aom/CMakeLists.txt +index 9dbe301..20c7be4 100644 +--- a/third_party/aom/CMakeLists.txt ++++ b/third_party/aom/CMakeLists.txt +@@ -56,6 +56,10 @@ option(BUILD_SHARED_LIBS "CMake should generate a shared library build." OFF) + + project(AOM C CXX) + ++set(CMAKE_C_FLAGS "-msimd128 -msse2 -msse3 -msse4.1 -msse4.2 ${CMAKE_C_FLAGS}") ++set(CMAKE_CXX_FLAGS "-msimd128 -msse2 -msse3 -msse4.1 -msse4.2 ${CMAKE_CXX_FLAGS}") ++set(CMAKE_VERBOSE_MAKEFILE on) ++ + set(AOM_ROOT "${CMAKE_CURRENT_SOURCE_DIR}") + set(AOM_CONFIG_DIR "${CMAKE_CURRENT_BINARY_DIR}") + set(INCLUDE_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/include" +@@ -347,7 +351,7 @@ if(CONFIG_AV1_DECODER AND ENABLE_EXAMPLES) + em_link_post_js(inspect "${AOM_ROOT}/tools/inspect-post.js") + # Force generation of Wasm instead of asm.js + append_link_flag_to_target("inspect" "-s WASM=1") +- append_compiler_flag("-s WASM=1") ++ append_compiler_flag("-O3 -s WASM=1 -s ERROR_ON_UNDEFINED_SYMBOLS=0") + endif() + endif() + +diff --git a/third_party/aom/aom/src/aom_codec.c b/third_party/aom/aom/src/aom_codec.c +index dbd6fa5..a8d2a49 100644 +--- a/third_party/aom/aom/src/aom_codec.c ++++ b/third_party/aom/aom/src/aom_codec.c +@@ -132,6 +132,7 @@ void aom_internal_error(struct aom_internal_error_info *info, + info->detail[sz - 1] = '\0'; + } + ++ printf("##aom internal error: %s\n", info->detail); + if (info->setjmp) longjmp(info->jmp, info->error_code); + } + +diff --git a/third_party/aom/aom_dsp/grain_table.c b/third_party/aom/aom_dsp/grain_table.c +index 0d6a73f..4b05833 100644 +--- a/third_party/aom/aom_dsp/grain_table.c ++++ b/third_party/aom/aom_dsp/grain_table.c +@@ -293,6 +293,9 @@ aom_codec_err_t aom_film_grain_table_read( + return error_info->error_code; + } + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + aom_codec_err_t aom_film_grain_table_write( + const aom_film_grain_table_t *t, const char *filename, + struct aom_internal_error_info *error_info) { +@@ -305,7 +308,7 @@ aom_codec_err_t aom_film_grain_table_write( + return error_info->error_code; + } + +- if (!fwrite(kFileMagic, 8, 1, file)) { ++ if (!emcc_fwrite(kFileMagic, 8, 1, file)) { + aom_internal_error(error_info, AOM_CODEC_ERROR, + "Unable to write file magic"); + fclose(file); +diff --git a/third_party/aom/aomdec.c b/third_party/aom/aomdec.c +index 4addee8..f850147 100644 +--- a/third_party/aom/aomdec.c ++++ b/third_party/aom/aomdec.c +@@ -274,6 +274,9 @@ static void update_image_md5(const aom_image_t *img, const int planes[3], + } + } + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + static void write_image_file(const aom_image_t *img, const int *planes, + const int num_planes, FILE *file) { + int i, y; +@@ -287,7 +290,7 @@ static void write_image_file(const aom_image_t *img, const int *planes, + const int h = aom_img_plane_height(img, plane); + + for (y = 0; y < h; ++y) { +- fwrite(buf, bytes_per_sample, w, file); ++ emcc_fwrite(buf, bytes_per_sample, w, file); + buf += stride; + } + } +diff --git a/third_party/aom/aomenc.c b/third_party/aom/aomenc.c +index 64155b0..3ed5080 100644 +--- a/third_party/aom/aomenc.c ++++ b/third_party/aom/aomenc.c +@@ -59,9 +59,12 @@ static size_t wrap_fread(void *ptr, size_t size, size_t nmemb, FILE *stream) { + } + #define fread wrap_fread + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + static size_t wrap_fwrite(const void *ptr, size_t size, size_t nmemb, + FILE *stream) { +- return fwrite(ptr, size, nmemb, stream); ++ return emcc_fwrite(ptr, size, nmemb, stream); + } + #define fwrite wrap_fwrite + +diff --git a/third_party/aom/aomstats.c b/third_party/aom/aomstats.c +index 0cfeea2..6833776 100644 +--- a/third_party/aom/aomstats.c ++++ b/third_party/aom/aomstats.c +@@ -80,9 +80,12 @@ void stats_close(stats_io_t *stats, int last_pass) { + } + } + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + void stats_write(stats_io_t *stats, const void *pkt, size_t len) { + if (stats->file) { +- (void)fwrite(pkt, 1, len, stats->file); ++ (void)emcc_fwrite(pkt, 1, len, stats->file); + } else { + if (stats->buf.sz + len > stats->buf_alloc_sz) { + size_t new_sz = stats->buf_alloc_sz + 64 * 1024; +diff --git a/third_party/aom/av1/common/debugmodes.c b/third_party/aom/av1/common/debugmodes.c +index 868f341..c44258c 100644 +--- a/third_party/aom/av1/common/debugmodes.c ++++ b/third_party/aom/av1/common/debugmodes.c +@@ -89,10 +89,13 @@ void av1_print_modes_and_motion_vectors(AV1_COMMON *cm, const char *file) { + fclose(mvs); + } + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + void av1_print_uncompressed_frame_header(const uint8_t *data, int size, + const char *filename) { + FILE *hdrFile = fopen(filename, "w"); +- fwrite(data, size, sizeof(uint8_t), hdrFile); ++ emcc_fwrite(data, size, sizeof(uint8_t), hdrFile); + fclose(hdrFile); + } + +diff --git a/third_party/aom/av1/encoder/encoder.c b/third_party/aom/av1/encoder/encoder.c +index a557380..d709d26 100644 +--- a/third_party/aom/av1/encoder/encoder.c ++++ b/third_party/aom/av1/encoder/encoder.c +@@ -2799,6 +2799,9 @@ AV1_COMP *av1_create_compressor(AV1EncoderConfig *oxcf, + snprintf((H) + strlen(H), sizeof(H) - strlen(H), (T), (V)) + #endif // CONFIG_INTERNAL_STATS + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + void av1_remove_compressor(AV1_COMP *cpi) { + AV1_COMMON *cm; + unsigned int i; +@@ -2814,7 +2817,7 @@ void av1_remove_compressor(AV1_COMP *cpi) { + if (cpi->oxcf.pass != 1) { + fprintf(stderr, "Writing counts.stt\n"); + FILE *f = fopen("counts.stt", "wb"); +- fwrite(&aggregate_fc, sizeof(aggregate_fc), 1, f); ++ emcc_fwrite(&aggregate_fc, sizeof(aggregate_fc), 1, f); + fclose(f); + } + #endif // CONFIG_ENTROPY_STATS +@@ -3013,7 +3016,7 @@ void aom_write_yuv_frame_420(YV12_BUFFER_CONFIG *s, FILE *f) { + int h = s->y_height; + + do { +- fwrite(src, s->y_width, 1, f); ++ emcc_fwrite(src, s->y_width, 1, f); + src += s->y_stride; + } while (--h); + +@@ -3021,7 +3024,7 @@ void aom_write_yuv_frame_420(YV12_BUFFER_CONFIG *s, FILE *f) { + h = s->uv_height; + + do { +- fwrite(src, s->uv_width, 1, f); ++ emcc_fwrite(src, s->uv_width, 1, f); + src += s->uv_stride; + } while (--h); + +@@ -3029,7 +3032,7 @@ void aom_write_yuv_frame_420(YV12_BUFFER_CONFIG *s, FILE *f) { + h = s->uv_height; + + do { +- fwrite(src, s->uv_width, 1, f); ++ emcc_fwrite(src, s->uv_width, 1, f); + src += s->uv_stride; + } while (--h); + } +@@ -3121,7 +3124,7 @@ void aom_write_one_yuv_frame(AV1_COMMON *cm, YV12_BUFFER_CONFIG *s) { + uint16_t *src16 = CONVERT_TO_SHORTPTR(s->y_buffer); + + do { +- fwrite(src16, s->y_width, 2, yuv_rec_file); ++ emcc_fwrite(src16, s->y_width, 2, yuv_rec_file); + src16 += s->y_stride; + } while (--h); + +@@ -3129,7 +3132,7 @@ void aom_write_one_yuv_frame(AV1_COMMON *cm, YV12_BUFFER_CONFIG *s) { + h = s->uv_height; + + do { +- fwrite(src16, s->uv_width, 2, yuv_rec_file); ++ emcc_fwrite(src16, s->uv_width, 2, yuv_rec_file); + src16 += s->uv_stride; + } while (--h); + +@@ -3137,7 +3140,7 @@ void aom_write_one_yuv_frame(AV1_COMMON *cm, YV12_BUFFER_CONFIG *s) { + h = s->uv_height; + + do { +- fwrite(src16, s->uv_width, 2, yuv_rec_file); ++ emcc_fwrite(src16, s->uv_width, 2, yuv_rec_file); + src16 += s->uv_stride; + } while (--h); + +@@ -3146,7 +3149,7 @@ void aom_write_one_yuv_frame(AV1_COMMON *cm, YV12_BUFFER_CONFIG *s) { + } + + do { +- fwrite(src, s->y_width, 1, yuv_rec_file); ++ emcc_fwrite(src, s->y_width, 1, yuv_rec_file); + src += s->y_stride; + } while (--h); + +@@ -3154,7 +3157,7 @@ void aom_write_one_yuv_frame(AV1_COMMON *cm, YV12_BUFFER_CONFIG *s) { + h = s->uv_height; + + do { +- fwrite(src, s->uv_width, 1, yuv_rec_file); ++ emcc_fwrite(src, s->uv_width, 1, yuv_rec_file); + src += s->uv_stride; + } while (--h); + +@@ -3162,7 +3165,7 @@ void aom_write_one_yuv_frame(AV1_COMMON *cm, YV12_BUFFER_CONFIG *s) { + h = s->uv_height; + + do { +- fwrite(src, s->uv_width, 1, yuv_rec_file); ++ emcc_fwrite(src, s->uv_width, 1, yuv_rec_file); + src += s->uv_stride; + } while (--h); + +@@ -3241,16 +3244,16 @@ static int dump_one_image(AV1_COMMON *cm, + + // --- Y --- + for (h = 0; h < cm->height; ++h) { +- fwrite(&ref_buf->y_buffer[h * ref_buf->y_stride], 1, cm->width, f_ref); ++ emcc_fwrite(&ref_buf->y_buffer[h * ref_buf->y_stride], 1, cm->width, f_ref); + } + // --- U --- + for (h = 0; h < (cm->height >> 1); ++h) { +- fwrite(&ref_buf->u_buffer[h * ref_buf->uv_stride], 1, (cm->width >> 1), ++ emcc_fwrite(&ref_buf->u_buffer[h * ref_buf->uv_stride], 1, (cm->width >> 1), + f_ref); + } + // --- V --- + for (h = 0; h < (cm->height >> 1); ++h) { +- fwrite(&ref_buf->v_buffer[h * ref_buf->uv_stride], 1, (cm->width >> 1), ++ emcc_fwrite(&ref_buf->v_buffer[h * ref_buf->uv_stride], 1, (cm->width >> 1), + f_ref); + } + +@@ -4692,17 +4695,17 @@ static void dump_filtered_recon_frames(AV1_COMP *cpi) { + + // --- Y --- + for (h = 0; h < cm->height; ++h) { +- fwrite(&recon_buf->y_buffer[h * recon_buf->y_stride], 1, cm->width, ++ emcc_fwrite(&recon_buf->y_buffer[h * recon_buf->y_stride], 1, cm->width, + f_recon); + } + // --- U --- + for (h = 0; h < (cm->height >> 1); ++h) { +- fwrite(&recon_buf->u_buffer[h * recon_buf->uv_stride], 1, (cm->width >> 1), ++ emcc_fwrite(&recon_buf->u_buffer[h * recon_buf->uv_stride], 1, (cm->width >> 1), + f_recon); + } + // --- V --- + for (h = 0; h < (cm->height >> 1); ++h) { +- fwrite(&recon_buf->v_buffer[h * recon_buf->uv_stride], 1, (cm->width >> 1), ++ emcc_fwrite(&recon_buf->v_buffer[h * recon_buf->uv_stride], 1, (cm->width >> 1), + f_recon); + } + +diff --git a/third_party/aom/av1/encoder/firstpass.c b/third_party/aom/av1/encoder/firstpass.c +index bb73fde..b963043 100644 +--- a/third_party/aom/av1/encoder/firstpass.c ++++ b/third_party/aom/av1/encoder/firstpass.c +@@ -476,6 +476,9 @@ static double raw_motion_error_stdev(int *raw_motion_err_list, + return raw_err_stdev; + } + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + #define UL_INTRA_THRESH 50 + #define INVALID_ROW -1 + void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { +@@ -1077,7 +1080,7 @@ void av1_first_pass(AV1_COMP *cpi, const struct lookahead_entry *source) { + else + recon_file = fopen(filename, "ab"); + +- (void)fwrite(lst_yv12->buffer_alloc, lst_yv12->frame_size, 1, recon_file); ++ (void)emcc_fwrite(lst_yv12->buffer_alloc, lst_yv12->frame_size, 1, recon_file); + fclose(recon_file); + } + +diff --git a/third_party/aom/build/cmake/aom_configure.cmake b/third_party/aom/build/cmake/aom_configure.cmake +index 9220a32..fb8bf9f 100644 +--- a/third_party/aom/build/cmake/aom_configure.cmake ++++ b/third_party/aom/build/cmake/aom_configure.cmake +@@ -260,7 +260,7 @@ if(MSVC) + add_compiler_flag_if_supported("/WX") + endif() + else() +- require_c_flag("-std=c99" YES) ++ #require_c_flag("-std=c99" YES) + add_compiler_flag_if_supported("-Wall") + add_compiler_flag_if_supported("-Wdisabled-optimization") + add_compiler_flag_if_supported("-Wextra") +diff --git a/third_party/aom/examples/resize_util.c b/third_party/aom/examples/resize_util.c +index 5485691..e60ed86 100644 +--- a/third_party/aom/examples/resize_util.c ++++ b/third_party/aom/examples/resize_util.c +@@ -45,6 +45,9 @@ static int parse_dim(char *v, int *width, int *height) { + return 1; + } + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + int main(int argc, char *argv[]) { + char *fin, *fout; + FILE *fpin, *fpout; +@@ -111,7 +114,7 @@ int main(int argc, char *argv[]) { + av1_resize_frame420(inbuf, width, inbuf_u, inbuf_v, width / 2, height, + width, outbuf, target_width, outbuf_u, outbuf_v, + target_width / 2, target_height, target_width); +- fwrite(outbuf, target_width * target_height * 3 / 2, 1, fpout); ++ emcc_fwrite(outbuf, target_width * target_height * 3 / 2, 1, fpout); + f++; + } + printf("%d frames processed\n", f); +diff --git a/third_party/aom/examples/scalable_encoder.c b/third_party/aom/examples/scalable_encoder.c +index 10d647e..fcf31e1 100644 +--- a/third_party/aom/examples/scalable_encoder.c ++++ b/third_party/aom/examples/scalable_encoder.c +@@ -91,6 +91,9 @@ void usage_exit(void) { + exit(EXIT_FAILURE); + } + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + static int encode_frame(aom_codec_ctx_t *codec, aom_image_t *img, + int frame_index, int flags, FILE *outfile) { + int got_pkts = 0; +@@ -105,7 +108,7 @@ static int encode_frame(aom_codec_ctx_t *codec, aom_image_t *img, + + if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) { + const int keyframe = (pkt->data.frame.flags & AOM_FRAME_IS_KEY) != 0; +- if (fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile) != ++ if (emcc_fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile) != + pkt->data.frame.sz) { + die_codec(codec, "Failed to write compressed frame"); + } +diff --git a/third_party/aom/ivfenc.c b/third_party/aom/ivfenc.c +index 80f4d14..d0e4e34 100644 +--- a/third_party/aom/ivfenc.c ++++ b/third_party/aom/ivfenc.c +@@ -14,6 +14,9 @@ + #include "aom/aom_encoder.h" + #include "aom_ports/mem_ops.h" + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + void ivf_write_file_header(FILE *outfile, const struct aom_codec_enc_cfg *cfg, + unsigned int fourcc, int frame_cnt) { + char header[32]; +@@ -32,7 +35,7 @@ void ivf_write_file_header(FILE *outfile, const struct aom_codec_enc_cfg *cfg, + mem_put_le32(header + 24, frame_cnt); // length + mem_put_le32(header + 28, 0); // unused + +- fwrite(header, 1, 32, outfile); ++ emcc_fwrite(header, 1, 32, outfile); + } + + void ivf_write_frame_header(FILE *outfile, int64_t pts, size_t frame_size) { +@@ -41,12 +44,12 @@ void ivf_write_frame_header(FILE *outfile, int64_t pts, size_t frame_size) { + mem_put_le32(header, (int)frame_size); + mem_put_le32(header + 4, (int)(pts & 0xFFFFFFFF)); + mem_put_le32(header + 8, (int)(pts >> 32)); +- fwrite(header, 1, 12, outfile); ++ emcc_fwrite(header, 1, 12, outfile); + } + + void ivf_write_frame_size(FILE *outfile, size_t frame_size) { + char header[4]; + + mem_put_le32(header, (int)frame_size); +- fwrite(header, 1, 4, outfile); ++ emcc_fwrite(header, 1, 4, outfile); + } +diff --git a/third_party/aom/test/decode_perf_test.cc b/third_party/aom/test/decode_perf_test.cc +index 3c93e7d..2d364ae 100644 +--- a/third_party/aom/test/decode_perf_test.cc ++++ b/third_party/aom/test/decode_perf_test.cc +@@ -24,6 +24,11 @@ + + using ::testing::make_tuple; + ++extern "C" { ++ size_t ++ emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++} ++ + namespace { + + #define VIDEO_NAME 0 +@@ -153,7 +158,7 @@ class AV1NewEncodeDecodePerfTest + + // Write frame header and data. + ivf_write_frame_header(outfile_, out_frames_, pkt->data.frame.sz); +- ASSERT_EQ(fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile_), ++ ASSERT_EQ(emcc_fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile_), + pkt->data.frame.sz); + } + +diff --git a/third_party/aom/test/film_grain_table_test.cc b/third_party/aom/test/film_grain_table_test.cc +index 0688146..dbb8e6b 100644 +--- a/third_party/aom/test/film_grain_table_test.cc ++++ b/third_party/aom/test/film_grain_table_test.cc +@@ -5,6 +5,11 @@ + #include "av1/encoder/grain_test_vectors.h" + #include "test/video_source.h" + ++extern "C" { ++ size_t ++ emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++} ++ + void grain_equal(const aom_film_grain_t *expected, + const aom_film_grain_t *actual) { + EXPECT_EQ(expected->apply_grain, actual->apply_grain); +@@ -168,7 +173,7 @@ TEST_F(FilmGrainTableIOTest, ReadTruncatedFile) { + + std::string grain_file; + FILE *file = libaom_test::GetTempOutFile(&grain_file); +- fwrite("deadbeef", 8, 1, file); ++ emcc_fwrite("deadbeef", 8, 1, file); + fclose(file); + ASSERT_EQ(AOM_CODEC_ERROR, + aom_film_grain_table_read(&table, grain_file.c_str(), &error_)); +diff --git a/third_party/aom/test/resize_test.cc b/third_party/aom/test/resize_test.cc +index e1c4e9f..9c2bce8 100644 +--- a/third_party/aom/test/resize_test.cc ++++ b/third_party/aom/test/resize_test.cc +@@ -22,6 +22,11 @@ + // Enable(1) or Disable(0) writing of the compressed bitstream. + #define WRITE_COMPRESSED_STREAM 0 + ++extern "C" { ++ size_t ++ emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++} ++ + namespace { + + #if WRITE_COMPRESSED_STREAM +@@ -55,13 +60,13 @@ static void write_ivf_file_header(const aom_codec_enc_cfg_t *const cfg, + mem_put_le32(header + 24, frame_cnt); /* length */ + mem_put_le32(header + 28, 0); /* unused */ + +- (void)fwrite(header, 1, 32, outfile); ++ (void)emcc_fwrite(header, 1, 32, outfile); + } + + static void write_ivf_frame_size(FILE *const outfile, const size_t size) { + char header[4]; + mem_put_le32(header, static_cast<unsigned int>(size)); +- (void)fwrite(header, 1, 4, outfile); ++ (void)emcc_fwrite(header, 1, 4, outfile); + } + + static void write_ivf_frame_header(const aom_codec_cx_pkt_t *const pkt, +@@ -76,7 +81,7 @@ static void write_ivf_frame_header(const aom_codec_cx_pkt_t *const pkt, + mem_put_le32(header + 4, pts & 0xFFFFFFFF); + mem_put_le32(header + 8, pts >> 32); + +- (void)fwrite(header, 1, 12, outfile); ++ (void)emcc_fwrite(header, 1, 12, outfile); + } + #endif // WRITE_COMPRESSED_STREAM + +@@ -309,7 +314,7 @@ class ResizeInternalTestLarge : public ResizeTest { + + // Write frame header and data. + write_ivf_frame_header(pkt, outfile_); +- (void)fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile_); ++ (void)emcc_fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile_); + } + #endif + +@@ -608,7 +613,7 @@ class ResizeCspTest : public ResizeTest { + + // Write frame header and data. + write_ivf_frame_header(pkt, outfile_); +- (void)fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile_); ++ (void)emcc_fwrite(pkt->data.frame.buf, 1, pkt->data.frame.sz, outfile_); + } + #endif + +diff --git a/third_party/aom/test/y4m_test.cc b/third_party/aom/test/y4m_test.cc +index ad901d9..f24093f 100644 +--- a/third_party/aom/test/y4m_test.cc ++++ b/third_party/aom/test/y4m_test.cc +@@ -19,6 +19,11 @@ + #include "test/util.h" + #include "test/y4m_video_source.h" + ++extern "C" { ++ size_t ++ emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++} ++ + namespace { + + using std::string; +@@ -68,7 +73,7 @@ static void write_image_file(const aom_image_t *img, FILE *file) { + (plane ? (img->d_w + img->x_chroma_shift) >> img->x_chroma_shift + : img->d_w); + for (y = 0; y < h; ++y) { +- fwrite(buf, bytes_per_sample, w, file); ++ emcc_fwrite(buf, bytes_per_sample, w, file); + buf += stride; + } + } +diff --git a/third_party/aom/third_party/googletest/src/googletest/src/gtest.cc b/third_party/aom/third_party/googletest/src/googletest/src/gtest.cc +index 5a8932c..ac2c435 100644 +--- a/third_party/aom/third_party/googletest/src/googletest/src/gtest.cc ++++ b/third_party/aom/third_party/googletest/src/googletest/src/gtest.cc +@@ -146,6 +146,11 @@ + # define vsnprintf _vsnprintf + #endif // GTEST_OS_WINDOWS + ++extern "C" { ++ size_t ++ emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++} ++ + namespace testing { + + using internal::CountIf; +@@ -3867,7 +3872,7 @@ class ScopedPrematureExitFile { + // errors are ignored as there's nothing better we can do and we + // don't want to fail the test because of this. + FILE* pfile = posix::FOpen(premature_exit_filepath, "w"); +- fwrite("0", 1, 1, pfile); ++ emcc_fwrite("0", 1, 1, pfile); + fclose(pfile); + } + } +diff --git a/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.cc b/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.cc +index 84655d8..0004093 100644 +--- a/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.cc ++++ b/third_party/aom/third_party/libwebm/mkvmuxer/mkvwriter.cc +@@ -14,6 +14,11 @@ + #include <share.h> // for _SH_DENYWR + #endif + ++extern "C" { ++ size_t ++ emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++} ++ + namespace mkvmuxer { + + MkvWriter::MkvWriter() : file_(NULL), writer_owns_file_(true) {} +@@ -32,7 +37,7 @@ int32 MkvWriter::Write(const void* buffer, uint32 length) { + if (buffer == NULL) + return -1; + +- const size_t bytes_written = fwrite(buffer, 1, length, file_); ++ const size_t bytes_written = emcc_fwrite(buffer, 1, length, file_); + + return (bytes_written == length) ? 0 : -1; + } +diff --git a/third_party/aom/tools_common.c b/third_party/aom/tools_common.c +index 7abc20c..fbc30bc 100644 +--- a/third_party/aom/tools_common.c ++++ b/third_party/aom/tools_common.c +@@ -185,6 +185,9 @@ const AvxInterface *get_aom_decoder_by_fourcc(uint32_t fourcc) { + } + #endif // CONFIG_AV1_DECODER + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + void aom_img_write(const aom_image_t *img, FILE *file) { + int plane; + +@@ -197,7 +200,7 @@ void aom_img_write(const aom_image_t *img, FILE *file) { + int y; + + for (y = 0; y < h; ++y) { +- fwrite(buf, 1, w, file); ++ emcc_fwrite(buf, 1, w, file); + buf += stride; + } + } +diff --git a/third_party/aom/video_writer.c b/third_party/aom/video_writer.c +index 4e072c7..6b1ca54 100644 +--- a/third_party/aom/video_writer.c ++++ b/third_party/aom/video_writer.c +@@ -66,10 +66,13 @@ void aom_video_writer_close(AvxVideoWriter *writer) { + } + } + ++size_t ++emcc_fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); ++ + int aom_video_writer_write_frame(AvxVideoWriter *writer, const uint8_t *buffer, + size_t size, int64_t pts) { + ivf_write_frame_header(writer->file, pts, size); +- if (fwrite(buffer, 1, size, writer->file) != size) return 0; ++ if (emcc_fwrite(buffer, 1, size, writer->file) != size) return 0; + + ++writer->frame_count; + |